#!/usr/bin/env python3 from google.cloud import datastore from collections import namedtuple import time from XLIFFToXLSX import process_xliff_soup from XLIFFReader import findXLIFFFiles, parse_xliff_file from concurrent.futures import ThreadPoolExecutor from ansicolor import black from AutoTranslationIndexer import TextTagIndexer from DatastoreUtils import DatastoreChunkClient client = datastore.Client(project="watts-198422") executor = ThreadPoolExecutor(512) chunkClient = DatastoreChunkClient(client, executor) def export_lang_to_db(lang, filt): count = 0 indexer = TextTagIndexer(lang) for file in findXLIFFFiles("cache/{}".format(lang), filt=filt): # e.g. '1_high_priority_platform/about.donate.xliff' canonicalFilename = "/".join(file.split("/")[2:]) print(black(file, bold=True)) soup = parse_xliff_file(file) for entry in process_xliff_soup(soup, also_approved=True): indexer.add(entry.Source, entry.Translated, file.rpartition("/")[2], entry.IsApproved) # Stats count += 1 if count % 1000 == 0: print("Processed {} records".format(count))
def get_datastore_client(): global datastore_client if datastore_client is None: from google.cloud import datastore datastore_client = datastore.Client() return datastore_client
# this generates all base ratings for every units # to use you will need pymongo import re import os import json from google.cloud import datastore client = datastore.Client(project='monplan-api-dev') def handleSETU(unitCode): setuData = open("setuData.json", "r") inputData = json.loads(setuData.read()) for items in inputData: if (items['UnitCode'] == unitCode): return items return True return { 'enjoyScore': 3, 'enjoyResponse': 0, 'learnScore': 3, 'learnResponse': 0 } #if no data on code return base rating of 3, and response of 0 def generateLocationAndTimeObject(array): tempArray = [] for i in range(0, len(array)): #array i: [['Peninsula'], ['First semester 2017 (Day)']] location = array[i][0][0] time = array[i][1]
def __init__(self, project, bucket_name, **kwargs): super(DataStoreClient, self).__init__(project, bucket_name, **kwargs) self.datastore = datastore.Client(self.project) self.name = "datastore" self.batch = DataStoreManager(client=self.datastore)
def clone_client(client): return datastore.Client(project=client.project, namespace=client.namespace, credentials=client._credentials, _http=client._http)
def get_datastore_data(): client = datastore.Client() query = client.query(kind='msft-compete') results = list(query.fetch()) return results
def DatastoreClient(): return datastore.Client( project="sbt-endpoints" )
def upload_text(): file_name = request.form["text"] client = storage.Client() bucket = client.get_bucket('gee-nlp') blob = bucket.get_blob(file_name) downloaded_blob = blob.download_as_string() downloaded_blob_1 = downloaded_blob.decode('utf-8', 'ignore') downloaded_blob_1 = downloaded_blob_1.replace("\r\n\r\n", " ") text = downloaded_blob_1 # Analyse sentiment using Sentiment API call #sentiment = analyze_text_sentiment(text)[0].get('sentiment score') results = analyze_text_sentiment(text) print("results: ", results) sentiment = results["overallResults"].get('score') magnitude = results["overallResults"].get('magnitude') print("sentiment in upload_text(): ", sentiment) sentence_sentiment = results["sentence_sentiment"] classify_document = gcp_classify_text(downloaded_blob_1) print("classify_document: ", classify_document) df_sentiment = pd.DataFrame(sentence_sentiment) df_sentiment gcp_plot_sentiments(df_sentiment) # Assign a label based on the score overall_sentiment = 'unknown' if sentiment > 0: overall_sentiment = 'positive' if sentiment < 0: overall_sentiment = 'negative' if sentiment == 0: overall_sentiment = 'neutral' # Create a Cloud Datastore client. datastore_client = datastore.Client() # Fetch the current date / time. current_datetime = datetime.now() # The kind for the new entity. This is so all 'Sentences' can be queried. kind = "gee-nlp-demo" # Create the Cloud Datastore key for the new entity. key = datastore_client.key(kind, 'sample_task1') # Alternative to above, the following would store a history of all previous requests as no key # identifier is specified, only a 'kind'. Datastore automatically provisions numeric ids. # key = datastore_client.key(kind) # Construct the new entity using the key. Set dictionary values for entity entity = datastore.Entity(key) entity["file_name"] = file_name entity["timestamp"] = current_datetime entity["sentiment"] = overall_sentiment entity["classify_document"] = classify_document entity["magnitude"] = magnitude print("entity: ", entity) # Save the new entity to Datastore. datastore_client.put(entity) # Redirect to the home page. return redirect("/")
def get_from_datastore(config_key, env): ds = datastore.Client() product_key = ds.key('Settings', env) entity = ds.get(product_key) return entity[config_key]
def clear_listings_cache(request): datastore_client = datastore.Client() query = datastore_client.query(kind="Listing") results = list(query.fetch()) for listing in results: datastore_client.delete(listing.key)
from google.cloud import datastore from . import events import os PROJECT_NAME = os.getenv("PROJECT_NAME") if PROJECT_NAME is None: raise Exception("env_var PROJECT_NAME was not found") datastore_client = datastore.Client(PROJECT_NAME) def exists(task_id): key = datastore_client.key("tasks", task_id) entity = datastore_client.get(key) return entity is not None def create(task): with datastore_client.transaction(): key = datastore_client.key("tasks", task["id"]) task.managerState = "Active" entity = { "key": key, "exclude_from_indexes": ["id", "data", "attributes"], "data": task }
def get_all_testers(): # Instantiates a client client = datastore.Client() query = client.query(kind='Tester') query_iter = query.fetch() return query_iter
# We need to first prepare the DataStore for a new days data. # http://stackoverflow.com/questions/22445217/python-webbrowser-open-to-open-chrome-browser url = 'daily-stock-forecast.com/cleards' # MacOS #chrome_path = 'open -a /Applications/Google\ Chrome.app %s' # Windows # chrome_path = 'C:\Program Files (x86)\Google\Chrome\Application\chrome.exe %s' # Linux chrome_path = '/usr/bin/google-chrome %s' webbrowser.get(chrome_path).open(url) print('\nWaiting 30 sec to ensure DS Prep...') time.sleep(30) # Set the dataset from the command line parameters. #datastore.set_options(dataset="daily-stock-forecast") client = datastore.Client("daily-stock-forecast") top_5 = [] #Save each symbol into the datastore for i in np.arange(len(data.symbols)): if rank[data.symbols[i]] <= 100000: #try: #req = datastore.CommitRequest() #req.mode = datastore.CommitRequest.NON_TRANSACTIONAL #entity = req.mutation.insert_auto_id.add() # Create a new entity key. key = client.key('Forecast') entity = datastore.Entity(key) # Set the entity key with only one `path_element`: no parent.
import os from flask import Flask from google.cloud import datastore APP_CREATED_FLASK_1PATH = "Created Flask application in folder: {}" def create_app(config={}): if path := config.get('INSTANCE_PATH'): app = Flask(__name__, instance_path=path) else: app = Flask(__name__, instance_relative_config=True) app.logger.info(APP_CREATED_FLASK_1PATH.format(app.instance_path)) app.config.from_pyfile(os.path.join(app.instance_path, 'config.py')) app.config.update(config) if config["PROJECT"] is None: exit("No project ID specified in kwargs or config.py") with app.app_context(): app.db = datastore.Client(project=app.config["PROJECT"]) from jigs.routes import jig_bp app.register_blueprint(jig_bp) return app
result += toAsciiMap[ch] return result for word in words: translit = get_translit(word) word_translits[translit] = word def get_label_ge(label_translit): return word_translits[label_translit] from google.cloud import datastore ds = datastore.Client("ge-open-speech-recording") def label_dir(dir_name): files = os.listdir(dir_name) for file_name in files: key = os.path.splitext(file_name)[0] sound = ds.get(sound_key) if sound: print("Already classified:", key) continue try: wave_file = wave.open(dir_name + "/" + file_name, 'r') except: print("Error:", file_name) continue
def create_client(project_id): return datastore.Client(project_id)
def initialize_datastore_client(): global datastore_client if datastore_client is None: datastore_client = datastore.Client(PROJECT_ID)
from telegram import Bot, ChatAction, Update, InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton, \ ReplyKeyboardMarkup, ReplyKeyboardRemove from google.cloud import datastore from geopy import distance # Enable logging logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO) logger = logging.getLogger(__name__) # Define global vars and constants bot = Bot(token=os.environ["TELEGRAM_TOKEN"]) db = datastore.Client() # A dictionary of lists holding organizations as keys and members belonging to them as values (lists). # A user can be in several orgs organizations = defaultdict(list) # A dictionary holding members as keys and their preferences as values. # members['pytonist'] = # {'selected_org': 'Pythonists', 'travel_radius': '3', 'location': {'longitude': 37.742, 'latitude': 55.45}} members = {} def db_query_by_kind(kind): """Query the datastore by entity kind (e.g. Organization, Member). The result is a list of Entities. To get an entity's id do result[index].id. If the id is custom, do result[index].key.name To get entity's items do result[index].items(). It can be converted to list result[index]['members'].
def __init__(self, kind): self.client = datastore.Client() self.kind = kind
def get_client(): return datastore.Client(current_app.config['PROJECT_ID'])
def __init__(self): credentials = service_account.Credentials.from_service_account_file( './GoogleCloudStuff/MyoBasketball-6daa0c614684.json') # Instantiates a client self.datastore_client = datastore.Client(credentials=credentials)
import re import shutil import sys import tempfile import traceback from google.cloud import datastore, storage import requests import config import gsutil import wptreport _log = logging.getLogger(__name__) _datastore = datastore.Client() def _get_uploader_password(username): """Gets the password for an uploader. Datastore exceptions may be raised. Args: username: A username (string). Returns: A string, the password for this user. """ return _datastore.get(_datastore.key('Uploader', username))['Password']
def generate_client(): global client client = datastore.Client('dancedeets-hrd')
def setUp(self): EndToEndWithFakes.prometheus_port += 1 # The directory of files on the server rsync directory. self.rsync_data_dir = '/tmp/iupui_ndt/' # rmtree removes the directory, so we need to use shell here. self.addCleanup(lambda: os.system('rm -Rf /tmp/iupui_ndt/*')) # The directory that serves as a fake Google Cloud Storage bucket. self.cloud_upload_dir = '/tmp/cloud_storage_bucket/' os.makedirs(self.cloud_upload_dir) self.addCleanup(lambda: shutil.rmtree('/tmp/cloud_storage_bucket')) # Patch credentials to do nothing. The local datastore emulator doesn't # even look at creds. creds_patcher = mock.patch.object(gce, 'AppAssertionCredentials', return_value=EmulatorCreds()) creds_patcher.start() self.addCleanup(creds_patcher.stop) local_datastore_client = datastore.Client(project='mlab-sandbox', namespace='test', credentials=EmulatorCreds(), _http=requests.Session()) # Make datastore clients connect to the local datastore emulator datastore_client_patcher = mock.patch.object( datastore, 'Client', return_value=local_datastore_client) datastore_client_patcher.start() self.addCleanup(datastore_client_patcher.stop) # Empty out the local datastore emulator. query = local_datastore_client.query(kind='dropboxrsyncaddress') entities = query.fetch() for entity in entities: local_datastore_client.delete(entity.key) # Make an entirely mocked storage service self.mock_storage = mock.MagicMock() discovery_build_patcher = mock.patch.object( apiclient.discovery, 'build', return_value=self.mock_storage) discovery_build_patcher.start() self.addCleanup(discovery_build_patcher.stop) class FakeProgress(object): def __init__(self, status): self.status = status def progress(self): return self.status surrounding_testcase = self class FakeRequest(object): """Handle upload requests by copying the file.""" def __init__(self, **kwargs): self.index = -1 # pylint: disable=protected-access shutil.copy(kwargs['media_body']._filename, surrounding_testcase.cloud_upload_dir) # pylint: enable=protected-access self.return_values = [(FakeProgress(.5), None), (FakeProgress(1.0), mock.MagicMock())] def next_chunk(self): self.index += 1 return self.return_values[self.index] inserter = mock.MagicMock() inserter.insert.side_effect = FakeRequest self.mock_storage.objects.return_value = inserter # Keep filenames unique self.file_index = 0
parser.error('--storing in datastore can only be done when --model_name is among the supported models: {}.'.format(model_choices)) model_name = args['model_name'] project_id = os.getenv('PROJECT_ID') bucket_name = os.getenv('BUCKET_NAME') location = os.getenv('LOCATION') key_path = os.getenv('SA_KEY_PATH') dataset_name = os.getenv('BQ_DATASET_NAME') table_name = os.getenv('BQ_TABLE_NAME') credentials = service_account.Credentials.from_service_account_file(key_path) storage_client = storage.Client(credentials=credentials) datastore_client = datastore.Client(credentials=credentials) bq_client = bigquery.Client(credentials=credentials) if args.store_bigquery == 'True': start_time = time.time() populateBQ(bq_client=bq_client,storage_client=storage_client, bucket_name=bucket_name, dataset_name=dataset_name, table_name=table_name) total_time = time.time() - start_time logging.info( 'The export to BigQuery was completed successfully and took {} seconds.'.format(round(total_time, 1))) else: logging.info('The export to BigQuery was disable.') if args.store_datastore == 'True':
from functools import partial from copy import deepcopy from google.cloud import datastore, exceptions from enum import Enum from datetime import datetime, date, time import binascii """ Tiny wrapper around *google.appengine.api.datastore*. This just ensures that operations issued directly through the database-api doesn't interfere with ViURs internal caching. If you need skeletons anyway, query the database using skel.all(); its faster and is able to serve more requests from cache. """ __client__ = datastore.Client() # Consts KEY_SPECIAL_PROPERTY = "__key__" DATASTORE_BASE_TYPES = Union[None, str, int, float, bool, datetime, date, time] class SortOrder(Enum): Ascending = 1 Descending = 2 InvertedAscending = 3 InvertedDescending = 4 # Proxied Function / Classed Entity = datastore.Entity
def autocomplete(request): start_time = time.time() global DATASTORE_CLIENT global REDIS_CONN # Parse typed string from a client. typed_string = '' request_json = request.get_json() if request.args and 'term' in request.args: typed_string = request.args.get('term') elif request_json and 'term' in request_json: typed_string = request_json['term'] else: response = jsonify([{}]) response.headers.add('Access-Control-Allow-Origin', '*') return response if not REDIS_CONN: try: REDIS_CONN = redis.Redis(host=REDIS_CONFIG['host'], port=REDIS_CONFIG['port']) except redis.ConnectionError: error_msg = 'Error: Redis Server not available' response = jsonify([].append({ 'label': error_msg, 'value': error_msg })) response.headers.add('Access-Control-Allow-Origin', '*') return response redis_key = hashlib.md5(typed_string.encode('utf-8')).hexdigest() cached_results = REDIS_CONN.get(redis_key) if cached_results: matches = json.loads(cached_results) response = jsonify(matches) response.headers.add('Access-Control-Allow-Origin', '*') elapsed_time = (time.time() - start_time) * 1000 # Performance Measurement print('Processing time: {0:.0f} ms (cached) [ {1} ] => [ {2} ]'.format( elapsed_time, typed_string, matches[0]['label'])) return response # Initialize Datastore Client if not DATASTORE_CLIENT: try: # DATASTORE_CLIENT = datastore.Client(project = PROJECT_ID) DATASTORE_CLIENT = datastore.Client() except: import traceback traceback.print_exc() # Create query and get the results query = DATASTORE_CLIENT.query(kind=GCP_CLOUD_DATASTORE_KIND) query.add_filter('name', '>=', typed_string) query.order = ['name'] results = list(query.fetch(limit=COUNT)) # Create response data matches = [] for result in results: matches.append({'label': result['name'], 'value': result['name']}) response = jsonify(matches) REDIS_CONN.set(redis_key, json.dumps(matches)) response.headers.add('Access-Control-Allow-Origin', '*') elapsed_time = (time.time() - start_time) * 1000 # Performance Measurement print('Processing time: {0:.0f} ms [ {1} ] => [ {2} ]'.format( elapsed_time, typed_string, matches[0]['label'])) return response
def DatastoreClient(): return datastore.Client(project="wave16-joan")
'autoescape': True, 'block_start_string': '<%', 'block_end_string': '%>', 'variable_start_string': '%%', 'variable_end_string': '%%', 'comment_start_string': '<#', 'comment_end_string': '#>' } TEMPLATE = 'index.html' # Flask setup app = Flask(__name__) app.jinja_options = JINJA_OPTIONS firebase_app = firebase_admin.initialize_app() # GCP setup client = datastore.Client() curated_partial_keys = None log_client = glogging.Client() log_client.get_default_handler() log_client.setup_logging() @app.route('/get/mod') def get_mod(): """Handler for single-module get requests.""" global curated_partial_keys mute_id = request.args.get('mute') if mute_id: set_mute(mute_id)
def upload_photo(): photo = request.files["file"] # Create a Cloud Storage client. storage_client = storage.Client() # Get the bucket that the file will be uploaded to. bucket = storage_client.get_bucket(CLOUD_STORAGE_BUCKET) # Create a new blob and upload the file's content. blob = bucket.blob(photo.filename) blob.upload_from_string(photo.read(), content_type=photo.content_type) # Make the blob publicly viewable. blob.make_public() # Create a Cloud Vision client. vision_client = vision.ImageAnnotatorClient() # Use the Cloud Vision client to detect a face for our image. source_uri = "gs://{}/{}".format(CLOUD_STORAGE_BUCKET, blob.name) image = vision.Image(source=vision.ImageSource(gcs_image_uri=source_uri)) faces = vision_client.face_detection(image=image).face_annotations # If a face is detected, save to Datastore the likelihood that the face # displays 'joy,' as determined by Google's Machine Learning algorithm. if len(faces) > 0: face = faces[0] # Convert the likelihood string. likelihoods = [ "Unknown", "Very Unlikely", "Unlikely", "Possible", "Likely", "Very Likely", ] face_joy = likelihoods[face.joy_likelihood] else: face_joy = "Unknown" # Create a Cloud Datastore client. datastore_client = datastore.Client() # Fetch the current date / time. current_datetime = datetime.now() # The kind for the new entity. kind = "Faces" # The name/ID for the new entity. name = blob.name # Create the Cloud Datastore key for the new entity. key = datastore_client.key(kind, name) # Construct the new entity using the key. Set dictionary values for entity # keys blob_name, storage_public_url, timestamp, and joy. entity = datastore.Entity(key) entity["blob_name"] = blob.name entity["image_public_url"] = blob.public_url entity["timestamp"] = current_datetime entity["joy"] = face_joy # Save the new entity to Datastore. datastore_client.put(entity) # Redirect to the home page. return redirect("/")