def _login_to_gcs(): """Helper to ensure we have login credentials to GCS in boto.""" gcs_oauth2_boto_plugin.SetFallbackClientIdAndSecret( secrets.BIGQUERY_CLIENT_ID, secrets.BIGQUERY_CLIENT_SECRET) try: uri = boto.storage_uri('', 'gs') uri.get_all_buckets( headers={"x-goog-project-id": secrets.BIGQUERY_PROJECT_ID}) except: plugin = gcs_oauth2_boto_plugin oauth2_refresh_token = (plugin.oauth2_helper.OAuth2ApprovalFlow( plugin.oauth2_helper.OAuth2ClientFromBotoConfig( boto.config, "Oauth 2.0 User Account"), ['https://www.googleapis.com/auth/devstorage.full_control'], False)) try: boto.config.add_section('Credentials') except: pass boto.config.set("Credentials", "gs_oauth2_refresh_token", oauth2_refresh_token) with open(os.path.expanduser("~/.boto"), "w") as f: boto.config.write(f) return boto
import os import boto import gcs_oauth2_boto_plugin import shutil from django.conf import settings import hlprs as helpers import helpers as main_helpers IMAGES_EXT = ['.jpg', '.png', '.tiff', '.gif'] # connecting to google cloud gcs_oauth2_boto_plugin.SetFallbackClientIdAndSecret(settings.GS_CLIENT_ID, settings.GS_CLIENT_SECRET) def save_tmp_file(fileobj, filename, ext): if ext in IMAGES_EXT: f = open("/tmp/" + filename + ext, 'wb') shutil.copyfileobj(fileobj, f) f.close() helpers.resize_image(filename, ext) if ext != '.jpg': os.remove("/tmp/" + filename + ext) return '.jpg' if ext in ['.txt']: f = open("/tmp/" + filename + ext, 'w') shutil.copyfileobj(fileobj, f) f.close()
import httplib, sys from Queue import Queue import socket sock = socket.socket() sock.settimeout(3) import uuid # URI scheme for Cloud Storage. GOOGLE_STORAGE = 'gs' # URI scheme for accessing local files. LOCAL_FILE = 'file' CLIENT_ID = 'getyourclientidfromgooglecloudconsole' CLIENT_SECRET = 'clientsecret' project_name = "project_name" gcs_oauth2_boto_plugin.SetFallbackClientIdAndSecret(CLIENT_ID, CLIENT_SECRET) #delete report path & compressFileName per run after uploaded to cloud urls_to_test = [] audit_urls = [] concurrent = 400 q = Queue(concurrent * 2) def list_buckets(): buckets = [] uri = boto.storage_uri('', GOOGLE_STORAGE)
def process_image(image_id, full_url, thumb_url, source_url): import requests import orm import boto import gcs_oauth2_boto_plugin import tempfile import mimetypes import conf from PIL import Image as pimage from PIL import ImageFile ImageFile.LOAD_TRUNCATED_IMAGES = True import imagehash from hashtest import hash_image session = orm.Session() gcs_oauth2_boto_plugin.SetFallbackClientIdAndSecret( conf.client_id, conf.client_secret) fullbucket = boto.storage_uri(conf.fullbucket, 'gs').get_bucket() thumbbucket = boto.storage_uri(conf.thumbbucket, 'gs').get_bucket() # Fetch images print "%d: Starting" % image_id response = requests.get(source_url, stream=True) if not response.status_code == 200: session.query(orm.Image).filter(orm.Image.id == image_id).update( {'fetched': -1}) session.commit() return fulltemp = tempfile.NamedTemporaryFile() thumbtemp = tempfile.NamedTemporaryFile() for block in response.iter_content(4096): fulltemp.write(block) fulltemp.seek(0) himg = pimage.open(fulltemp) ahash, phash, dhash = imagehash.average_hash(himg), imagehash.phash( himg), imagehash.dhash(himg) ahash, phash, dhash = int(str(ahash), base=16), int(str(phash), base=16), int(str(dhash), base=16) # Save images, make thumb himg.thumbnail((640, 640)) himg.convert("RGB").save(thumbtemp, format='WebP') del himg if ahash >= 2**63: ahash -= 2**64 if phash >= 2**63: phash -= 2**64 if dhash >= 2**63: dhash -= 2**64 # Upload fulltemp.seek(0) thumbtemp.seek(0) fullkey = fullbucket.new_key(full_url.split('/')[-1]) thumbkey = thumbbucket.new_key(thumb_url.split('/')[-1]) meta = { 'Cache-Control': 'public, max-age=3600', 'Content-Type': response.headers['content-type'], } fullkey.set_contents_from_file(fulltemp, headers=meta) print "%d: Uploaded full" % image_id meta['Content-Type'] = 'image/webp' thumbkey.set_contents_from_file(thumbtemp, headers=meta) print "%d: Uploaded thumb" % image_id try: bmbhash = hash_image(fulltemp.name) session.add(orm.Hash(name=u'bmbhash', value=bmbhash, image_id=image_id)) except: pass session.add(orm.Hash(name=u'ahash', value=ahash, image_id=image_id)) session.add(orm.Hash(name=u'phash', value=phash, image_id=image_id)) session.add(orm.Hash(name=u'dhash', value=dhash, image_id=image_id)) session.query(orm.Image).filter(orm.Image.id == image_id).update({ 'fetched': 1, 'size': int(response.headers['content-length']) }) session.commit() fulltemp.close() thumbtemp.close()
def __init__(self): gcs_oauth2_boto_plugin.SetFallbackClientIdAndSecret( CONFIG['client_id'], CONFIG['client_secret'])
def configure_boto(client_id, client_secret): gcs_oauth2_boto_plugin.SetFallbackClientIdAndSecret( client_id, client_secret)