def _login_to_gcs():
"""Helper to ensure we have login credentials to GCS in boto."""
gcs_oauth2_boto_plugin.SetFallbackClientIdAndSecret(
secrets.BIGQUERY_CLIENT_ID, secrets.BIGQUERY_CLIENT_SECRET)
try:
uri = boto.storage_uri('', 'gs')
uri.get_all_buckets(
headers={"x-goog-project-id": secrets.BIGQUERY_PROJECT_ID})
except:
plugin = gcs_oauth2_boto_plugin
oauth2_refresh_token = (plugin.oauth2_helper.OAuth2ApprovalFlow(
plugin.oauth2_helper.OAuth2ClientFromBotoConfig(
boto.config, "Oauth 2.0 User Account"),
['https://www.googleapis.com/auth/devstorage.full_control'],
False))
try:
boto.config.add_section('Credentials')
except:
pass
boto.config.set("Credentials", "gs_oauth2_refresh_token",
oauth2_refresh_token)
with open(os.path.expanduser("~/.boto"), "w") as f:
boto.config.write(f)
return boto
import os
import boto
import gcs_oauth2_boto_plugin
import shutil
from django.conf import settings
import hlprs as helpers
import helpers as main_helpers
IMAGES_EXT = ['.jpg', '.png', '.tiff', '.gif']
# connecting to google cloud
gcs_oauth2_boto_plugin.SetFallbackClientIdAndSecret(settings.GS_CLIENT_ID,
settings.GS_CLIENT_SECRET)
def save_tmp_file(fileobj, filename, ext):
if ext in IMAGES_EXT:
f = open("/tmp/" + filename + ext, 'wb')
shutil.copyfileobj(fileobj, f)
f.close()
helpers.resize_image(filename, ext)
if ext != '.jpg':
os.remove("/tmp/" + filename + ext)
return '.jpg'
if ext in ['.txt']:
f = open("/tmp/" + filename + ext, 'w')
shutil.copyfileobj(fileobj, f)
f.close()
import httplib, sys
from Queue import Queue
import socket
sock = socket.socket()
sock.settimeout(3)
import uuid
# URI scheme for Cloud Storage.
GOOGLE_STORAGE = 'gs'
# URI scheme for accessing local files.
LOCAL_FILE = 'file'
CLIENT_ID = 'getyourclientidfromgooglecloudconsole'
CLIENT_SECRET = 'clientsecret'
project_name = "project_name"
gcs_oauth2_boto_plugin.SetFallbackClientIdAndSecret(CLIENT_ID, CLIENT_SECRET)
#delete report path & compressFileName per run after uploaded to cloud
urls_to_test = []
audit_urls = []
concurrent = 400
q = Queue(concurrent * 2)
def list_buckets():
buckets = []
uri = boto.storage_uri('', GOOGLE_STORAGE)
def process_image(image_id, full_url, thumb_url, source_url):
import requests
import orm
import boto
import gcs_oauth2_boto_plugin
import tempfile
import mimetypes
import conf
from PIL import Image as pimage
from PIL import ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True
import imagehash
from hashtest import hash_image
session = orm.Session()
gcs_oauth2_boto_plugin.SetFallbackClientIdAndSecret(
conf.client_id, conf.client_secret)
fullbucket = boto.storage_uri(conf.fullbucket, 'gs').get_bucket()
thumbbucket = boto.storage_uri(conf.thumbbucket, 'gs').get_bucket()
# Fetch images
print "%d: Starting" % image_id
response = requests.get(source_url, stream=True)
if not response.status_code == 200:
session.query(orm.Image).filter(orm.Image.id == image_id).update(
{'fetched': -1})
session.commit()
return
fulltemp = tempfile.NamedTemporaryFile()
thumbtemp = tempfile.NamedTemporaryFile()
for block in response.iter_content(4096):
fulltemp.write(block)
fulltemp.seek(0)
himg = pimage.open(fulltemp)
ahash, phash, dhash = imagehash.average_hash(himg), imagehash.phash(
himg), imagehash.dhash(himg)
ahash, phash, dhash = int(str(ahash),
base=16), int(str(phash),
base=16), int(str(dhash), base=16)
# Save images, make thumb
himg.thumbnail((640, 640))
himg.convert("RGB").save(thumbtemp, format='WebP')
del himg
if ahash >= 2**63:
ahash -= 2**64
if phash >= 2**63:
phash -= 2**64
if dhash >= 2**63:
dhash -= 2**64
# Upload
fulltemp.seek(0)
thumbtemp.seek(0)
fullkey = fullbucket.new_key(full_url.split('/')[-1])
thumbkey = thumbbucket.new_key(thumb_url.split('/')[-1])
meta = {
'Cache-Control': 'public, max-age=3600',
'Content-Type': response.headers['content-type'],
}
fullkey.set_contents_from_file(fulltemp, headers=meta)
print "%d: Uploaded full" % image_id
meta['Content-Type'] = 'image/webp'
thumbkey.set_contents_from_file(thumbtemp, headers=meta)
print "%d: Uploaded thumb" % image_id
try:
bmbhash = hash_image(fulltemp.name)
session.add(orm.Hash(name=u'bmbhash', value=bmbhash,
image_id=image_id))
except:
pass
session.add(orm.Hash(name=u'ahash', value=ahash, image_id=image_id))
session.add(orm.Hash(name=u'phash', value=phash, image_id=image_id))
session.add(orm.Hash(name=u'dhash', value=dhash, image_id=image_id))
session.query(orm.Image).filter(orm.Image.id == image_id).update({
'fetched':
1,
'size':
int(response.headers['content-length'])
})
session.commit()
fulltemp.close()
thumbtemp.close()
def __init__(self):
gcs_oauth2_boto_plugin.SetFallbackClientIdAndSecret(
CONFIG['client_id'], CONFIG['client_secret'])
def configure_boto(client_id, client_secret):
gcs_oauth2_boto_plugin.SetFallbackClientIdAndSecret(
client_id, client_secret)