class GoogleCloudFile(File):
def __init__(self, name, mode, storage):
self.name = name
self.mime_type = mimetypes.guess_type(name)[0]
self._mode = mode
self._storage = storage
self.blob = storage.bucket.get_blob(name)
if not self.blob and 'w' in mode:
self.blob = Blob(self.name,
storage.bucket,
chunk_size=storage.blob_chunk_size)
self._file = None
self._is_dirty = False
@property
def size(self):
return self.blob.size
def _get_file(self):
if self._file is None:
self._file = SpooledTemporaryFile(
max_size=self._storage.max_memory_size,
suffix=".GSStorageFile",
dir=setting("FILE_UPLOAD_TEMP_DIR"))
if 'r' in self._mode:
self._is_dirty = False
self.blob.download_to_file(self._file)
self._file.seek(0)
return self._file
def _set_file(self, value):
self._file = value
file = property(_get_file, _set_file)
def read(self, num_bytes=None):
if 'r' not in self._mode:
raise AttributeError("File was not opened in read mode.")
if num_bytes is None:
num_bytes = -1
return super().read(num_bytes)
def write(self, content):
if 'w' not in self._mode:
raise AttributeError("File was not opened in write mode.")
self._is_dirty = True
return super().write(force_bytes(content))
def close(self):
if self._file is not None:
if self._is_dirty:
self.blob.upload_from_file(
self.file,
rewind=True,
content_type=self.mime_type,
predefined_acl=self._storage.default_acl)
self._file.close()
self._file = None
def upload_image_to_gcs(self, image_file_path, bucket_name):
"""Uploads images to Google Cloud Storage.
Arguments:
file_name : Name of image to upload on local machine
image_file_path: Path to image to upload on local machine.
bucket_name: Name of the GCS bucket.
"""
project = "<GCP_project_id>"
credentials = service_account.Credentials.from_service_account_file(
'~/gcp-service-account.json')
client = storage.Client(project, credentials)
try:
bucket = client.get_bucket(bucket_name)
except Exception as e:
# Error out if we're unable to locate the S3 bucket.
raise MotionAlertError("Error connecting to GCS bucket: "
"{0}".format(e))
try:
temp_image = image_file_path.split('/')
image_file = temp_image[3] + "/" + temp_image[4]
blob = Blob(image_file, bucket)
with open(image_file_path, 'rb') as my_file:
blob.upload_from_file(my_file)
except Exception as e:
# Error out if we're unable to upload the image.
raise MotionAlertError(
"Error uploading file to GCS: {0}".format(e))
def _upload(self, payload: bytes, filename: str, bucket: str) -> None:
"""
Upload a payload to GCS
"""
client = Client(project=self.project_id)
count = 0
while count < self.max_retries:
try:
bucket_obj = client.get_bucket(bucket)
if self.use_encryption:
payload = self._encrypt(payload)
content = BytesIO(payload)
blob = Blob(filename, bucket_obj)
blob.upload_from_file(content)
break
except (
InvalidResponse,
GoogleAPICallError,
InternalServerError,
SSLError,
) as e:
if count >= self.max_retries:
raise StoqPluginException(
f'Failed to upload {bucket}/{filename} to GCS: {str(e)}'
)
count += 1
sleep(randrange(0, 4))
def upload_image(self, location: str, label: str, image_bytes,
metadata: dict) -> typing.Optional[str]:
blob = Blob(f"{location}/{label}", self.bucket)
blob.metadata = metadata
blob.upload_from_file(image_bytes, content_type="image/png")
blob.make_public()
return blob.public_url
def download_bhavcopy(event, context):
holiday_dict = create_holiday_dict()
logging.info('Dictionary created for ' + str(len(holiday_dict)) +
' holidays')
base_url = 'https://www.bseindia.com/download/BhavCopy/Equity/'
pubsub_message = base64.b64decode(event['data']).decode('utf-8')
print(pubsub_message)
print(event['attributes'])
fname = event['attributes']['objectId']
extracted_date = re.search(r'([eE][qQ])(\d\d\d\d\d\d)', fname).group(2)
new_date = datetime.strptime(extracted_date, '%d%m%y') + timedelta(days=1)
file_downloaded_locally, new_fname = check_and_download(
new_date, holiday_dict, base_url)
try:
if file_downloaded_locally and (
not check_if_already_stored(new_fname)):
client = storage.Client(project='bhavcopy')
bucket = client.get_bucket('bhavcopy-store')
blob = Blob(new_fname, bucket)
with open('/tmp/' + new_fname, 'rb') as my_file:
blob.upload_from_file(my_file)
except Exception as e:
logging.info(
'Not Downloaded: Cloud function exiting without storing file for date: '
+ str(new_date) + '.Received error: ' + str(e))
def upload_item_as_raw_file(path, client=None):
"""Set things up, convert the file, and upload it."""
if client is None:
client = get_storage_client()
# Check that the bucket exists, make it if not.
try:
b = client.get_bucket(TRANSCRIPTS_BUCKET_NAME)
except Forbidden as e:
print("Received Forbidden (403) error while getting bucket. This could "
"mean that you do not have billing set up for this "
"account/project, or that somebody else has taken this bucket "
"from the global namespace.")
raise e
except NotFound:
b = client.bucket(TRANSCRIPTS_BUCKET_NAME)
b.lifecycle_rules = [{
'action': {'type': 'Delete'},
'condition': {'age': 7},
}]
b.create()
b.make_public(future=True)
# Re-encode the file as a temp file and upload it. When we leave the context
# manager, the temp file gets automatically deleted.
with NamedTemporaryFile(prefix='transcode_', suffix='.raw') as tmp:
encode_as_linear16(path, tmp)
# Name it after a SHA2 hash of the item, to avoid collisions.
file_name = 'transcripts-%s' % hashlib.sha256(tmp.read()).hexdigest()
blob = Blob(file_name, b)
blob.upload_from_file(tmp, rewind=True)
return {'blob_name': blob.name, 'bucket_name': blob.bucket.name}
def upload_arr(arr: np.ndarray, blob: storage.Blob):
logging.info(f'uploading blob {blob.name}')
with io.BytesIO() as stream:
# noinspection PyTypeChecker
np.save(stream, arr)
stream.seek(0)
blob.upload_from_file(stream)
def write(self,
file_path,
data,
num_retries=10,
content_type=None,
bucket=None):
bucket = self.client.get_bucket(self.bucket or bucket)
try:
blob = Blob(file_path, bucket)
except:
blob = bucket.get_blob(file_path)
try:
data = json.loads(data)
except:
pass
if isinstance(data,(dict,list)):
data = json.dumps(data)
else:
data = data
data = _to_bytes(data, encoding="utf-8")
string_buffer = BytesIO(data)
blob.upload_from_file(
file_obj = string_buffer,
size = len(data),
client = self.client,
num_retries = num_retries or self.num_retries,
content_type = _MEME_TYPES[self.content_type or content_type]
)
return
def put(self, source, name=None):
"""
Parameters
----------
source
name
Returns
-------
"""
if isinstance(source, io.BytesIO):
filebuff = io.BufferedReader(source)
elif isinstance(source, (str, bytes)):
filebuff = io.BufferedReader(io.BytesIO(source))
else:
log.error(
'Source should be either a string, or bytes or io.BytesIO, got {}'
.format(type(source)))
return False
key = self.base_address if name is None else os.path.join(
self.base_address, name)
key = key.strip('/')
try:
blob = Blob(key, self.bucket)
blob.upload_from_file(filebuff, rewind=True)
log.info("Uploaded {} bytes to \tbucket={}\tkey={}".format(
len(source), self.bucket_name, key))
return True
except Exception as e:
log.error("{}\tbucket={}\tkey={}".format(e, self.bucket, key))
return False
def write_gcloud_blob(bucket_id: str, gpath: str, fpath: str):
""" Write blob from Google Cloud Storage.
References:
https://pypi.org/project/google-cloud-storage/
:param bucket_id: id for google cloud bucket
:param gpath: file path of item within bucket
:param fpath: file path of item from disk
:return: upload file blob from disk
:rtype: None
"""
logger.info("Writing '{}' to '{}' at '{}'".\
format(fpath, bucket_id, gpath))
try:
client = storage.Client()
bucket = client.get_bucket(bucket_id)
blob = Blob(gpath, bucket)
with open(fpath, 'rb') as infile:
blob.upload_from_file(infile)
logger.info("SUCCESS -- uploaded '{}' to '{}' using '{}'".\
format(fpath, gpath, bucket_id))
except Exception as exc:
logger.error("Unable to upload '{}' to '{}' using '{}'".\
format(fpath, gpath, bucket_id))
logger.exception(exc)
def upload_static_files(bucket, file_name):
# Uploader Form
blob = Blob(name=file_name, bucket=bucket)
with open(os.path.abspath('../static-files/%s' % file_name), 'rb') as tmp_file:
blob.upload_from_file(tmp_file)
# Resource
print(u'Info: File %s uploaded' % file_name)
print(blob)
def file_upload(file, request_date):
if file and allowed_file(file.filename):
new_file_name = request_date.replace(" ", "-").replace(
":", "-") + "_" + file.filename
safename = secure_filename(new_file_name)
blob = Blob(safename, get_storage())
blob.upload_from_file(file)
print('File uploaded successfully')
def upload(filename):
client = storage.Client(project=PROJECT_NAME)
bucket = client.get_bucket(BUCKET_NAME)
blob_name = "{}_{}".format(str(datetime.datetime.now()), filename)
blob = Blob(blob_name, bucket)
with open(filename, 'rb') as my_file:
blob.upload_from_file(my_file)
blob_q.put("gs://{}/{}".format(BUCKET_NAME, blob_name))
def upload_public_file(client, bkt, file_name):
# file_name in Blob constructor is the file name you want to have on GCS
blob = Blob(file_name, bkt)
# file_name in open function is the one that actually sits on your hard drive
with open(file_name, 'rb') as my_file:
blob.upload_from_file(my_file)
# after uploading the blob, we set it to public, so that it's accessible with a simple link
blob.make_public(client)
def download_html_company_information(company_symbols, bucket_name):
"""Downloads HTML files with company information and stores localy in the VM and upload in Cloud Storage"""
client = storage.Client()
bucket = client.get_bucket(bucket_name)
# Set the Display for the VM to download the HTML COde
display = Display(visible=0, size=(800, 600))
display.start()
print "Downloading the Company Information HTML files for: " + str(
len(company_symbols)) + " symbols"
company_symbols = ["AXON"]
i = 1
for symbol in company_symbols:
print "Downloading symbol " + str(i) + ": " + str(
symbol) + " from the list"
# Download locally in a vm specified folder
url = "http://financials.morningstar.com/company-profile/c.action?t=" + symbol + "®ion=GRC&culture=en_US"
#htmlFile = urllib2.urlopen(url)
htmlFilePath = "/home/project_investing/datasources/html_" + symbol + "_" + datetime.datetime.today(
).strftime('%Y%m%d') + ".html"
driver = webdriver.Chrome()
#options = webdriver.ChromeOptions()
#options.binary_location = '/opt/google/chrome/google-chrome'
#service_log_path = "{}/chromedriver.log".format("/home/project_investing")
#service_args = ['--verbose']
#driver = webdriver.Chrome('/usr/bin/chromedriver', chrome_options=options, service_args=service_args, service_log_path=service_log_path)
driver.get(url)
#waiting for the page to load - TODO: change
#wait = WebDriverWait(driver, 10)
#wait.until(EC.visibility_of_element_located((By.ID, "content")))
data = driver.page_source
with open(htmlFilePath, 'wb') as output:
output.write(data)
driver.close()
# Upload to Cloud Storage
blob_fileName = "datasources/html_files/html_" + symbol + "_" + datetime.datetime.today(
).strftime('%Y%m%d') + ".html"
blob = Blob(blob_fileName, bucket)
with open(htmlFilePath, 'rb') as input:
blob.upload_from_file(input)
i += 1
print "All HTML files have been succesfully stored in Cloud Storage in the following path: gs://athens_stock_exchange/datasources/html_files/"
def uploadNewPhoto(self, file, bucket):
blob = Blob(file, bucket)
# TODO: create csv/tuple for the input?
with open(file, "rb") as my_file:
blob.upload_from_file(my_file)
blob.make_public()
return r"gs://" + bucket.name + r"/" + file
def store_file_to_gcs(self, bucket_name, filename):
bucket = self.gcs.get_bucket(bucket_name)
blob = Blob(filename, bucket)
try:
with open(filename, 'rb') as input_file:
blob.upload_from_file(input_file)
return True
except IOError:
print('Error: Cannot find the file {}'.format(filename))
def copyFromLocal(self,
uri_source,
uri_target,
bucket_name_target=None,
**kwargs):
gcp_bucket_target = self._gcp_bucket(bucket_name_target)
blob = Blob(uri_target, gcp_bucket_target)
with open(uri_source, 'rb') as file_pointer:
blob.upload_from_file(file_pointer)
def upload_file(module, client, src, dest):
try:
bucket = client.get_bucket(module.params['bucket'])
blob = Blob(dest, bucket)
with open(src, "r") as file_obj:
blob.upload_from_file(file_obj)
return blob_to_dict(blob)
except google.cloud.exceptions.GoogleCloudError as e:
module.fail_json(msg=str(e))
def get_gcloud_url(stream):
filename = ''.join(
random.choice(string.ascii_lowercase + string.digits)
for i in range(16)) + ".png"
client = storage.Client()
bucket = client.get_bucket('hackgt-catalyst2018-photostyle')
blob = Blob(filename, bucket)
blob.upload_from_file(stream, content_type="image/png", client=client)
blob.make_public(client=client)
return blob.public_url
def upload_data(data):
formatted_version = '_'.join(factorio_version().split('.'))
storage_filename = 'factorio-data/v{}.json.gz'.format(formatted_version)
data_blob = Blob(storage_filename, storage.bucket(app=actuario_app))
data_blob.content_encoding = 'gzip'
data_gz_bytes = gzip.compress(json.dumps(data).encode())
with io.BytesIO(data_gz_bytes) as data_stream:
data_blob.upload_from_file(data_stream,
content_type='application/json')
def main():
#
# initial greeting...
#
print("Hello Google Cloud Storage!")
#
# create a client
#
print("creating client...")
client = storage.Client()
index = 0
print("indexing over bucket list...")
for bucket in client.list_buckets():
print(bucket)
print("index = " + str(index))
if index == 0:
defaultBucket = bucket
index += 1
print("")
print("chosen bucket is: " + str(defaultBucket))
blob = Blob("raw_image.jpg", defaultBucket)
quit = False
imageFilePath = "/home/shawn/Desktop/raw_image_download.jpg"
while quit == False:
blobCount = 0
for blobItem in defaultBucket.list_blobs():
blobCount += 1
if blobCount == 0:
print("empty...")
else:
print("downloading...")
with open(imageFilePath, "wb") as imageFile:
blob.download_to_file(imageFile)
with Image(filename=imageFilePath) as img:
print(img.size)
print("blurring...")
img.gaussian_blur(9, 1)
imageFilePath = "/home/shawn/Desktop/blurred_image.jpg"
print("saving...")
img.save(filename=imageFilePath)
with Image(filename=imageFilePath) as img:
blob = Blob("blurred_image.jpg", defaultBucket)
print("uploading...")
with open("/home/shawn/Desktop/blurred_image.jpg",
"rb") as imageFile:
blob.upload_from_file(imageFile)
display(img)
time.sleep(1.0)
#
# final greeting...
#
print("Goodbye Google Cloud Storage!")
def upload_from_file(client, to_delete):
# [START upload_from_file]
from google.cloud.storage import Blob
client = storage.Client(project='my-project')
bucket = client.get_bucket('my-bucket')
encryption_key = 'aa426195405adee2c8081bb9e7e74b19'
blob = Blob('secure-data', bucket, encryption_key=encryption_key)
with open('my-file', 'rb') as my_file:
blob.upload_from_file(my_file)
# [END upload_from_file]
to_delete.append(blob)
def upload_from_file(client, to_delete):
# [START upload_from_file]
from google.cloud.storage import Blob
client = storage.Client(project='my-project')
bucket = client.get_bucket('my-bucket')
encryption_key = 'aa426195405adee2c8081bb9e7e74b19'
blob = Blob('secure-data', bucket, encryption_key=encryption_key)
with open('my-file', 'rb') as my_file:
blob.upload_from_file(my_file)
# [END upload_from_file]
to_delete.append(blob)
def upload_from_file(to_delete):
# [START upload_from_file]
from google.cloud.storage import Blob
client = storage.Client(project="my-project")
bucket = client.get_bucket("my-bucket")
encryption_key = "aa426195405adee2c8081bb9e7e74b19"
blob = Blob("secure-data", bucket, encryption_key=encryption_key)
with open("my-file", "rb") as my_file:
blob.upload_from_file(my_file)
# [END upload_from_file]
to_delete.append(blob)
def _save(self, name, content):
name = os.path.basename(name)
new_name = name
count = 0
while True:
blob = Blob(new_name, self.bucket, chunk_size=1024 * 256)
if not blob.exists():
break
count += 1
new_name = name + '.%d' % count
blob.upload_from_file(content)
blob.make_public()
return new_name
def upload_from_file(client, to_delete):
# [START upload_from_file]
from google.cloud.storage import Blob
client = storage.Client(project="my-project")
bucket = client.get_bucket("my-bucket")
encryption_key = "aa426195405adee2c8081bb9e7e74b19"
blob = Blob("secure-data", bucket, encryption_key=encryption_key)
with open("my-file", "rb") as my_file:
blob.upload_from_file(my_file)
# [END upload_from_file]
to_delete.append(blob)
def write(self, file_path, data, num_retries=10, content_type=None, bucket=None):
"""Writes data to Google Cloud Storage
:type data: bytes
:param data: The data that will be written to Google Cloud Storage
:type file_path: str
:param file_path: The path to the file where the data will be written
:type num_retries: int
:param num_retries: (Optional) The number of attempts to save to Cloud Storage
:type content_type: str
:param content_type: The type of content being uploaded. Default is 'json'. Options are txt, csv, json, png, jpg
:type bucket: str
:param bucket: The name of the bucket. (Optional) if given in class instantiation.
:returns: None
"""
bucket = self.client.get_bucket(self.bucket or bucket)
try:
blob = Blob(file_path, bucket)
except:
blob = bucket.get_blob(file_path)
try:
data = json.loads(data)
except:
pass
if isinstance(data, (dict, list)):
data = json.dumps(data)
else:
data = data
data = _to_bytes(data, encoding="utf-8")
string_buffer = BytesIO(data)
blob.upload_from_file(
file_obj=string_buffer,
size=len(data),
client=self.client,
num_retries=num_retries or self.num_retries,
content_type=_MEME_TYPES[self.content_type or content_type],
)
return
def upload():
"""Process the uploaded file and upload it to Google Cloud Storage."""
uploaded_file = request.files['file']
client = storage.Client()
bucket = client.get_bucket(request.form['uploadbucketname'])
encoded_key = base64.b64encode(key).decode('utf-8')
encryption_key = base64.b64decode(encoded_key)
blob = Blob(uploaded_file.filename, bucket, encryption_key=encryption_key)
blob.upload_from_file(uploaded_file)
return blob.public_url
def upload_blob(file_path, input_file):
ct = time.time()
logging.info('Start uploading file')
"""Uploads a file to the bucket."""
client = storage.Client(project=UPLOAD_GOOGLE_PROJECT)
bucket = client.get_bucket(UPLOAD_GOOGLE_BUCKET_NAME)
if is_dev_env:
file_path = os.path.join(VERSION, UPLOAD_TEST_PATH, file_path)
else:
file_path = os.path.join(VERSION, file_path)
blob = Blob(file_path, bucket)
input_file.seek(0)
blob.upload_from_file(input_file)
duration = time.time() - ct
logging.info('Finished uploading file %s', duration)
def on_http_fetch_dailies(request):
gcp_logger = init_gcp_logger()
gcp_logger.info('on_http_fetch_dailies(), args=%s', request.args)
storage_client = init_storage_client()
fdb = init_firestore_client()
columns = [
'meter_consumptions_kwh', 'meter_generations_kwh',
'solar_generations_kwh', 'solar_mean_powrs_kw',
'solar_devices_reportings', 'capacities_kw', 'charge_quantities_kwh',
'deterioration_states_pct', 'discharge_quantities_kwh',
'power_at_charges_kw', 'residual_capacities_pct',
'total_charge_quantities_kwh', 'total_discharge_quantities_kwh',
'min_temperature_c', 'max_temperature_c'
]
df_all_dailies = pd.DataFrame(index=pd.DatetimeIndex([]), columns=columns)
missing_values = np.zeros(48)
for doc in fdb.collection(f"sites/{NMI}/dailies").order_by(
'interval_date', direction='ASCENDING').stream():
doc_fields = list(doc.to_dict().keys())
if 'interval_date' not in doc_fields:
gcp_logger.info('Missing field interval_date')
continue
gcp_logger.info('Processing interval_date=%s',
doc.get('interval_date'))
doc_dict = {}
for column in columns:
doc_dict[column] = np.array(
doc.get(column)) if column in doc_fields else missing_values
df_all_dailies.loc[doc.get('interval_date')] = doc_dict
pkl_file_name = f"dailies_{NMI}.pkl"
pkl_file_path = f"/tmp/{pkl_file_name}"
df_all_dailies.to_pickle(f"/tmp/{pkl_file_name}")
bucket = storage_client.get_bucket(GCP_STORAGE_BUCKET_ID)
blob = Blob(pkl_file_name, bucket)
with open(pkl_file_path, "rb") as pkl_file:
blob.upload_from_file(pkl_file)
os.remove(pkl_file_path)
return ('', 200)
def upload_file():
if request.method == 'POST':
# check if the post request has the file part
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
file = request.files['file']
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
flash('No selected file')
return redirect(request.url)
if file and allowed_file(file.filename):
print('allowed_file')
filename = secure_filename(file.filename)
blob = Blob(filename, BUCKET)
blob.upload_from_file(file)
return redirect(url_for('uploaded_file', filename=filename))
return '''
