def __init__(self, project, bucket_name, tempdir='/tmp'):
# connect to the cloud bucket
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = os.environ.get(
'privatekey_path')
self.client = storage.Client(project)
self.bucket = self.client.get_bucket(bucket_name)
self.tempdir = tempdir
def _join(self, filename='exit_code', delete=True):
"""Wait until all calculation are finished:
This means wait until every bucket has an file with the name of the given filename
:param str filename: The filename where the exit code is stored
:param bool delete: Should the instance be deleted after calculation
"""
finished_tasks = []
logger.info("Calculating..")
gcs = storage.Client(self.project.project_id)
while len(finished_tasks) < len(self.job.tasks):
for t in [
task for task in self.job.tasks
if task not in finished_tasks
]:
bucket = gcs.get_bucket(t.id)
if bucket.get_blob(filename):
finished_tasks.append(t)
logger.info(
"Calculation is finished for instance: {}".format(
t.id))
if delete:
self._delete_instances([t])
time.sleep(10)
logger.info("Calculation is finished")
def delete_user_image(user_id):
# Check to see if the user exists
u = User.get_by_id(user_id)
if u is None:
raise InvalidUsage('UserID does not match any existing user',
status_code=400)
path = u.profile_picture_path
if path is None:
raise InvalidUsage('User has no profile picture.', status_code=400)
# Create client for interfacing with Cloud Storage API
client = storage.Client()
bucket = client.get_bucket(global_vars.USER_IMG_BUCKET)
bucket.delete_blob(path)
u.profile_picture_path = None
u.put()
now = datetime.datetime.now()
# Return response
resp = jsonify({'picture_id deleted': path, 'date_deleted': now})
resp.status_code = 200
return resp
def price(result):
client = storage.Client()
bucket = client.get_bucket('123iof')
bucket.blob("updatedcsfc.xls")
dfne = pd.read_excel('gs://123iof/updatedcfsc.xls')
dfne = pd.DataFrame(dfne)
dfne['ProductID'] = dfne["ProductID"].astype(str)
currentdt = datetime.datetime.now()
#serial_obj.close()
global newlocdf
newlocdf = dfne.loc[dfne['ProductID'] == str(result)]
time = newlocdf['Time'].values
temp = newlocdf['temperature'].values
layout = go.Layout(title='Days vs Temperature',
xaxis=dict(title='Days'),
yaxis=dict(title='Temperature'))
trace = go.Scatter(x=time, y=temp)
data = [trace]
fig = go.Figure(data=data, layout=layout)
graphJSON = json.dumps(fig, cls=plotly.utils.PlotlyJSONEncoder)
return graphJSON
def list_blobs_with_prefix(bucket_name, prefix, delimiter=None):
"""Lists all the blobs in the bucket that begin with the prefix.
This can be used to list all blobs in a "folder", e.g. "public/".
The delimiter argument can be used to restrict the results to only the
"files" in the given "folder". Without the delimiter, the entire tree under
the prefix is returned. For example, given these blobs:
/a/1.txt
/a/b/2.txt
If you just specify prefix = '/a', you'll get back:
/a/1.txt
/a/b/2.txt
However, if you specify prefix='/a' and delimiter='/', you'll get back:
/a/1.txt
"""
storage_client = storage.Client()
bucket = storage_client.get_bucket(bucket_name)
blobs = bucket.list_blobs(prefix=prefix, delimiter=delimiter)
print('Blobs:')
for blob in blobs:
print(blob.name)
if delimiter:
print('Prefixes:')
for prefix in blobs.prefixes:
print(prefix)
def combine():
client = storage.Client(PROJECT_ID)
bucket = client.bucket(PROJECT_ID + ".appspot.com")
blobs = bucket.list_blobs()
os.system("rm /tmp/*")
names = []
for blob in blobs:
if "output" in blob.name:
names.append(blob.name.encode('utf-8'))
names.sort()
with open('/tmp/combine.lst', 'w') as f1:
for name in names:
f1.write("file '/tmp/" + name + "'\n")
download(name)
logger.log_text("Worker: created combine list: /tmp/combine.lst")
ret = os.system(
"ffmpeg -f concat -safe 0 -i /tmp/combine.lst -c copy /tmp/combined.mkv"
)
if ret:
logger.log_text("Worker: combine failed: /tmp/combine.mkv - " +
str(ret).encode('utf-8'))
return
upload("combined.mkv")
def setUpModule():
Config.CLIENT = storage.Client()
# %d rounds milliseconds to nearest integer.
bucket_name = 'new%d' % (1000 * time.time(),)
# In the **very** rare case the bucket name is reserved, this
# fails with a ConnectionError.
Config.TEST_BUCKET = Config.CLIENT.create_bucket(bucket_name)
def get_fpl_data(req: Request) -> str:
"""Cloud function to save data from FPL website.
Writes raw JSON to Google Cloud storage.
Args:
req: Flask Request automatically passed by gcp Functions. Is ignored.
Returns:
Filename that was written to Google Cloud storage
"""
del req
storage_client = storage.Client()
logging.info("Beginning FPL data collection run.")
response_content = _fpl_downloader()
fpl_bucket = storage_client.bucket(FPL_BUCKET_NAME)
fpl_blob_name = 'fpl_data_' + dt.datetime.utcnow().isoformat()
new_blob = fpl_bucket.blob(fpl_blob_name)
new_blob.upload_from_file(BytesIO(response_content), size=2048)
logging.info(f"Data uploaded to {new_blob.name}.")
return new_blob.name
def update_file_cache(settings):
''' download JSON files from GCP Storage bucket, returns a list of the
files that were downloaded/changed '''
etags = load_json(settings['cache.dir'] + '/gcp/etags.json')
#FIXME
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = settings['creds.dir'] + \
"/" + \
settings['creds.gcp.json']
credentials = GoogleCredentials.get_application_default()
client = storage.Client(project=settings['creds.gcp.project'],
credentials=credentials)
bucket = client.get_bucket('exported-billing')
LOG.debug("Checking for new/changed files.")
changed = []
for obj in bucket.list_blobs():
filename = settings['cache.dir'] + '/gcp/' + obj.name
if not os.path.exists(filename) or \
filename not in etags or \
obj.etag != etags[filename]:
try:
LOG.debug("Etags for %s: %s == %s", obj.name, obj.etag,
etags[filename])
except KeyError:
LOG.debug("Etag missing: %s", obj.name)
LOG.info("Downloading: %s", obj.name)
obj.download_to_filename(filename)
etags[filename] = obj.etag
changed.append(os.path.basename(filename))
save_json(settings['cache.dir'] + '/gcp/etags.json', etags)
return changed
def new_listing_image(listing_id):
# user_id = request.form['user_id']
# listing_id = request.form['listing_id']
userfile = request.files['userfile']
filename = userfile.filename
# Check if listing exists
l = Listing.get_by_id(listing_id)
if l is None:
raise InvalidUsage('Listing does not exist!', status_code=400)
# Create client for interfacing with Cloud Storage API
client = storage.Client()
bucket = client.get_bucket(global_vars.LISTING_IMG_BUCKET)
# Calculating size this way is not very efficient. Is there another way?
userfile.seek(0, 2)
size = userfile.tell()
userfile.seek(0)
# upload the item image
path = str(listing_id) + '/' + filename
image = bucket.blob(blob_name=path)
image.upload_from_file(file_obj=userfile,
size=size,
content_type='image/jpeg')
# Hacky way of making the image public..
image.acl.all().grant_read()
image.acl.save()
resp = jsonify({'image_path': path, 'image_media_link': image.media_link})
resp.status_code = 201
return resp
def post(self, request, *args, **kwargs):
serializer = self.get_serializer(data=request.data)
if not serializer.is_valid():
return FailedResponse(status_message='Invalid request')
report = Report.objects.filter(id=request.data["report_id"]).first()
if not report:
return FailedResponse(status_message='Report not found')
credentials_dict = json.loads(base64.b64decode(settings.GCP_AUTH))
credentials = ServiceAccountCredentials.from_json_keyfile_dict(
credentials_dict)
client = storage.Client(credentials=credentials,
project=settings.GCP_PROJECT_ID)
bucket = client.get_bucket(settings.GCP_BUCKET_NAME)
image = base64.b64decode(serializer.data.get('photo'))
record = ReportPhoto(user_id=request.user.id,
report_id=request.data["report_id"])
record.save()
blob = bucket.blob(f'images/{record.id}.jpg', chunk_size=262144)
blob.upload_from_string(image)
blob.make_public()
record.public_url = blob.public_url
record.save()
return SuccessResponse(status_message='Success',
data={
"photo_id": record.id,
"public_url": blob.public_url
})
def upload_gcp_func(file, filename):
credentials_dict = {
"type":
"service_account",
"project_id":
"hackator",
"private_key_id":
"<enter your private key id>",
"private_key":
"<enter private key>",
"client_email":
"*****@*****.**",
"client_id":
"<enter client id>",
"auth_uri":
"https://accounts.google.com/o/oauth2/auth",
"token_uri":
"https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url":
"https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url":
"https://www.googleapis.com/robot/v1/metadata/x509/koushikhack44%40hackator.iam.gserviceaccount.com"
}
credentials = ServiceAccountCredentials.from_json_keyfile_dict(
credentials_dict)
client = storage.Client(credentials=credentials, project='hackator')
bucket = client.get_bucket('kbuckethack')
blob = bucket.blob(filename)
blob.upload_from_string(file)
def CategoryGenerator(bucket, file_path):
client = storage.Client()
bucket = client.get_bucket(bucket)
src_url_file = bucket.get_blob(file_path)
src_url_string = src_url_file.download_as_string()
for category_name in src_url_string.decode('utf-8').split('\r\n'):
yield Category(category_name)
def load(context, path, callback):
if path.startswith('http'): #_use_http_loader(context, url):
logger.debug("[LOADER] load with http_loader")
http_loader.load_sync(context,
path,
callback,
normalize_url_func=http_loader._normalize_url)
return
bucket_id = context.config.get("CLOUD_STORAGE_BUCKET_ID")
project_id = context.config.get("CLOUD_STORAGE_PROJECT_ID")
bucket = buckets[project_id].get(bucket_id, None)
logger.debug("[LOADER] loading from bucket")
if bucket is None:
client = storage.Client(project_id)
bucket = client.get_bucket(bucket_id)
buckets[project_id][bucket_id] = bucket
blob = bucket.get_blob(path)
if blob:
callback(blob.download_as_string())
else:
callback(blob)
def upload_images():
try:
user_id = request.authorization.get('username')
file = request.files['uploaded_file']
filename = file.filename
#folder = 'app/static/images/{0}/'.format(user_id)
#completeName = folder+filename
#dir = os.path.dirname(completeName)
#if not os.path.exists(dir):
# os.makedirs(dir)
#file.save(completeName)
#file.close()
completeName = "{0}-{1}".format(user_id, filename)
# Create a Cloud Storage client.
gcs = storage.Client()
# Get the bucket that the file will be uploaded to.
bucket = gcs.get_bucket(CLOUD_STORAGE_BUCKET)
# Create a new blob and upload the file's content.
blob = bucket.blob(completeName)
blob.upload_from_string(file.read(), content_type=file.content_type)
print "after file is uploaded::{0}".format(blob.public_url)
# The public URL can be used to directly access the uploaded file via HTTP.
return make_response(jsonify({'url': blob.public_url}), 200)
except Exception, e:
logging.error(str(e))
abort(400)
def testMoviePy():
clips_array = []
client = storage.Client('set-cloud-gaston')
bucket = client.bucket('set-cloud-gaston.appspot.com')
blob = bucket.blob("MM41916160062.mp4")
tempFile = "/tmp/MM41916160062.mp4"
print("testMoviePy - about to download {}".format(tempFile))
with open(tempFile, 'w') as f:
blob.download_to_file(f)
print("testMoviePy - downloaded {}".format(tempFile))
rgb = VideoFileClip(tempFile)
collor_bars_1 = rgb.subclip(0, 150)
content = rgb.subclip(150, 1419)
collor_bars_2 = rgb.subclip(1419, 1517)
print("testMoviePy - clip extraction done ")
clips_array.append(content)
print("testMoviePy - about to contatenate ")
final = concatenate_videoclips(clips_array)
print("testMoviePy - about to write video /tmp/MM41916160062_edited.mp4")
final.write_videofile("/tmp/MM41916160062_edited.mp4")
print("testMoviePy - /tmp/MM41916160062_edited.mp4 file created")
blob = bucket.blob('MM41916160062_edited.mp4')
print(
"testMoviePy - about to upload to GC video as MM41916160062_edited.mp4"
)
blob.upload_from_file(open('/tmp/MM41916160062_edited.mp4'))
print("testMoviePy - done succesfully")
def setUpModule():
Config.CLIENT = storage.Client()
bucket_name = 'new' + unique_resource_id()
# In the **very** rare case the bucket name is reserved, this
# fails with a ConnectionError.
Config.TEST_BUCKET = Config.CLIENT.bucket(bucket_name)
retry_429(Config.TEST_BUCKET.create)()
def transcode(messageAsString):
#testMoviePy()
print('inside def transcode(messageAsString) line 10')
client = storage.Client('set-cloud-gaston')
bucket = client.bucket('set-cloud-gaston.appspot.com')
blob = bucket.blob(messageAsString)
tempFile = "/tmp/" + messageAsString
with open(tempFile, 'w') as f:
print("about to download {}".format(tempFile))
blob.download_to_file(f)
print("{} downloaded".format(tempFile))
# os.system('rm /tmp/output.webm')
command = '/usr/bin/avconv -i ' + tempFile + ' -c:v libvpx -crf 10 -b:v 1M -c:a libvorbis /tmp/output.webm'
print('executing OS command')
print(command)
ret = os.system(command)
if ret:
sys.stderr.write("FAILED")
print('OS command failed')
return "Failed"
print('OS command finished successfully')
blob = bucket.blob('youtube_demo.webm')
blob.upload_from_file(open('/tmp/output.webm'))
sys.stdout.write("SUCCESS")
return "SUCCESS"
def _get_client(project_name):
if project_name not in GS._clients:
if storage is None:
raise FileSystemNotSupported(
'Google Storage is not supported. Install "gcloud".')
GS._clients[project_name] = storage.Client(project_name)
return GS._clients[project_name]
def start():
#change this for the credentials
os.environ[
"GOOGLE_APPLICATION_CREDENTIALS"] = r"C:\\Users\\vince\Desktop\\MyGoogleCloudService\\linear-equator-253121-2b458fe691e7.json"
#instantiating a client with a corresponded bucket
client = storage.Client()
bucketName = input("Enter Your Bucket:")
try:
bucket = client.get_bucket(bucketName)
except Exception as e:
print(e)
bucketPath = input(
"Enter Your disired Bucket Path(Format= \'[path]/\')\nIf no path, press Enter: "
)
#path for all the aduio files
path = glob("../../data/audio/*.flac")
#file name for google cloud
fileNameGC = [
bucketPath + name.replace("../../data/audio\\", "") for name in path
]
#creating a tuple for the threadpool
tuples = []
for i in range(len(path)):
tup = (fileNameGC[i], path[i])
tuples.append(tup)
for tup in tuples:
printResult(upload(tup, bucket))
print("All uploads are finished")
def createdatabase():
dfnew1 = dfnew[['Time', 'temperature', 'ItemName', 'price']]
conn = sqlite3.connect('sense8.db')
cursor = conn.cursor()
print("Opened database successfully")
try:
dfnew1.to_sql('producttable', conn)
dfnew.to_sql('sensortablefinalnew1', conn)
except ValueError:
print('already exists')
sql = ''' INSERT INTO sensortablefinalnew1(ProductID,Time,temperature,ItemName,price,shelflife,calories,cholestrol_in_mg,pottasium,protiens,vitamin_A,vitamin_C,Calcium,Iron)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)'''
#cur = conn.commit()
#cur = conn.cursor()
sql2 = ''' INSERT INTO producttable(ProductID,Time,temperature,ItemName,price)
VALUES(?,?,?,?,?)'''
conn.executemany(sql, dfnew.to_records(index=True, ))
conn.executemany(sql2, dfnew1.to_records(index=True, ))
conn.commit()
global dfsql
dfsql = pd.read_sql_query("SELECT * FROM sensortablefinalnew1", conn)
conn.close()
dfsql.to_excel("cfscnew.xls")
client = storage.Client()
bucket = client.get_bucket('123iof')
bucket.blob("newcfsc.xls")
blob = bucket.blob('updatedcfsc.xls')
with open('cfscnew.xls', 'rb') as csv:
blob.upload_from_file(csv)
return dfsql
def main():
connGCS = GCStorage(storage.Client())
connBQ = GCBQ(bigquery.Client())
bucketId = 'bigqbucket110211'
datasetId = 'universal110211'
tableId = 'UniversalBank'
mybucket = connGCS.getGCBucket(connGCS.client, bucketId)
#print(mybucket)
if mybucket is None:
mybucket = connGCS.createGCBucket(connGCS.client, bucketId)
fileCopied = connGCS.copylocaltoGCPBucket(file_to_load_final, mybucket)
if fileCopied:
print('File Copied to GCP Bucket {}', format(mybucket))
#print(mybucket.list_blobs())
dataset_id = connBQ.getBQDsetId(datasetId, connBQ.client)
#print(dataset_id)
if dataset_id is None:
dataset_id = connBQ.createDatasetBQ("{}.{}".format(connBQ.client.project, datasetId), connBQ.client)
#print(dataset_id)
table_id = connBQ.getBQTableId("{}.{}.{}".format(connBQ.client.project, dataset_id, tableId), connBQ.client)
#print(table_id)
if table_id is not None:
table_id = connBQ.createTableBQ(tableId, datasetId, connBQ.client)
#conn.getFileURI(mybucket)
#print(mybucket.get_blob("{}.csv".format(tableId)).public_url)
fileLoadedToBQ = connBQ.loaddataIntoBQ(connBQ.client, file_to_load_final, mybucket, tableId, datasetId, file_to_load_final)
def blob_metadata(bucket_name, blob_name):
"""Prints out a blob's metadata."""
storage_client = storage.Client()
bucket = storage_client.get_bucket(bucket_name)
blob = bucket.get_blob(blob_name)
print('Blob: {}'.format(blob.name))
print('Bucket: {}'.format(blob.bucket.name))
print('Storage class: {}'.format(blob.storage_class))
print('ID: {}'.format(blob.id))
print('Size: {} bytes'.format(blob.size))
print('Updated: {}'.format(blob.updated))
print('Generation: {}'.format(blob.generation))
print('Metageneration: {}'.format(blob.metageneration))
print('Etag: {}'.format(blob.etag))
print('Owner: {}'.format(blob.owner))
print('Component count: {}'.format(blob.component_count))
print('Crc32c: {}'.format(blob.crc32c))
print('md5_hash: {}'.format(blob.md5_hash))
print('Cache-control: {}'.format(blob.cache_control))
print('Content-type: {}'.format(blob.content_type))
print('Content-disposition: {}'.format(blob.content_disposition))
print('Content-encoding: {}'.format(blob.content_encoding))
print('Content-language: {}'.format(blob.content_language))
print('Metadata: {}'.format(blob.metadata))
def get_sdrf_info(project_id, bucket_name, disease_codes, header,
set_index_col, search_patterns):
client = storage.Client(project_id)
bucket = client.get_bucket(bucket_name)
# connect to google cloud storage
gcs = GcsConnector(project_id, bucket_name)
sdrf_info = pd.DataFrame()
for disease_code in disease_codes:
for blob in bucket.list_blobs(prefix=disease_code):
sdrf_filename = blob.name
if not all(x in sdrf_filename for x in search_patterns):
continue
print(sdrf_filename)
filebuffer = gcs.download_blob_to_file(sdrf_filename)
# convert to a dataframe
sdrf_df = convert_file_to_dataframe(filebuffer, skiprows=0)
sdrf_df = cleanup_dataframe(sdrf_df)
sdrf_df['Study'] = disease_code
try:
sdrf_df = sdrf_df.set_index(set_index_col)
except:
sdrf_df = sdrf_df.set_index("Derived_Array_Data_File")
sdrf_info = sdrf_info.append(sdrf_df)
print("Done loading SDRF files.")
return sdrf_info
def prepare_gs_default_test_bucket():
# Check credentials are present: this procedure should not be
# called otherwise.
if no_real_gs_credentials():
assert False
bucket_name = bucket_name_mangle('waletdefwuy', delimiter='')
conn = storage.Client()
def _clean():
bucket = conn.get_bucket(bucket_name)
for blob in bucket.list_blobs():
try:
bucket.delete_blob(blob.path)
except exceptions.NotFound:
pass
try:
conn.create_bucket(bucket_name)
except exceptions.Conflict:
# Conflict: bucket already present. Re-use it, but
# clean it out first.
pass
_clean()
return bucket_name
def print_bucket_acl(bucket_name):
"""Prints out a bucket's access control list."""
storage_client = storage.Client()
bucket = storage_client.bucket(bucket_name)
for entry in bucket.acl:
print('{}: {}'.format(entry['role'], entry['entity']))
def __init__(self, project_name, bucket_name, credential_path):
self.client = storage.Client(
credentials=GCStorage.get_credentials(credential_path),
project=project_name)
self.bucket = self.client.get_bucket(bucket_name)
self.bucket_name = bucket_name
GCStorage.MONO = self
def test_blob(cloud_config):
"""Provides a pre-existing blob in the test bucket."""
bucket = storage.Client().bucket(cloud_config.storage_bucket)
blob = bucket.blob('encryption_test_sigil')
content = 'Hello, is it me you\'re looking for?'
blob.upload_from_string(content,
encryption_key=TEST_ENCRYPTION_KEY_DECODED)
return blob.name, content
def __init__(self, credentials, storage_bucket, requests):
self.storage_bucket = "https://firebasestorage.googleapis.com/v0/b/" + storage_bucket
self.credentials = credentials
self.requests = requests
self.path = ""
if credentials:
client = storage.Client(credentials=credentials, project=storage_bucket)
self.bucket = client.get_bucket(storage_bucket)
def to_storage(bucket, bucket_folder, file_name, path_to_file):
client = storage.Client(project="gavinete-sv")
bucket = client.get_bucket(f"{bucket}")
blob = bucket.blob(f"{bucket_folder}/{file_name}")
blob.upload_from_filename(f"{path_to_file}")
print("Done!")
