def test_blob():
"""Provides a pre-existing blob in the test bucket."""
bucket = storage.Client().bucket(BUCKET)
blob = Blob('encryption_test_sigil',
bucket, encryption_key=TEST_ENCRYPTION_KEY_DECODED)
content = 'Hello, is it me you\'re looking for?'
blob.upload_from_string(content)
return blob.name, content
def download_to_file(client, to_delete):
# [START download_to_file]
from google.cloud.storage import Blob
client = storage.Client(project='my-project')
bucket = client.get_bucket('my-bucket')
encryption_key = 'c7f32af42e45e85b9848a6a14dd2a8f6'
blob = Blob('secure-data', bucket, encryption_key=encryption_key)
blob.upload_from_string('my secret message.')
with open('/tmp/my-secure-file', 'wb') as file_obj:
blob.download_to_file(file_obj)
# [END download_to_file]
to_delete.append(blob)
def download_to_file(client, to_delete):
# [START download_to_file]
from google.cloud.storage import Blob
client = storage.Client(project="my-project")
bucket = client.get_bucket("my-bucket")
encryption_key = "c7f32af42e45e85b9848a6a14dd2a8f6"
blob = Blob("secure-data", bucket, encryption_key=encryption_key)
blob.upload_from_string("my secret message.")
with open("/tmp/my-secure-file", "wb") as file_obj:
blob.download_to_file(file_obj)
# [END download_to_file]
to_delete.append(blob)
def get_data():
"""
Retrieve the data file from GCP Storage, and return
the file as a dictionary.
Create the file, with dummy data, if it don't exist.
"""
# Introduce a delay here.
do_delay()
# Start of the actual function
rtn = None
storage_client = storage.Client()
bucket_name = current_app.config.get('DATA_BUCKET_NAME')
print('bucket_name=%s' % bucket_name)
try:
bucket = storage_client.get_bucket(bucket_name)
except Exception as e:
bucket = storage_client.create_bucket(bucket_name)
# Test if the data file is found in the bucket, and
# create it if it doesn't exist.
blob = Blob(current_app.config.get('DATA_FILE_NAME'), bucket)
if not blob.exists():
# Open the initial data file
init_fname = current_app.config.get('INIT_DATA_FILE')
with open(init_fname) as infile:
init_data = json.load(infile)
# Copy it to the storage bucket
blob.upload_from_string(json.dumps(init_data, indent=4))
data_str = blob.download_as_string()
rtn = json.loads(data_str)
print('GOT BAG DATA:')
print(json.dumps(rtn, indent=4))
return rtn, blob
def write_data(self,
data: str,
bucket_name: str,
name: str,
content_type: str = None):
blob = Blob(name, self.get_bucket(bucket_name))
blob.upload_from_string(data, content_type=content_type)
def handle_bq_lock(gcs_client: storage.Client, lock_blob: storage.Blob,
next_job_id: Optional[str]):
"""Reclaim the lock blob for the new job id (in-place) or delete the lock
blob if next_job_id is None."""
try:
if next_job_id:
if lock_blob.exists(client=gcs_client):
lock_blob.upload_from_string(
next_job_id,
if_generation_match=lock_blob.generation,
client=gcs_client)
else: # This happens when submitting the first job in the backlog
lock_blob.upload_from_string(next_job_id,
if_generation_match=0,
client=gcs_client)
else:
print("releasing lock at: "
f"gs://{lock_blob.bucket.name}/{lock_blob.name}")
lock_blob.delete(
if_generation_match=lock_blob.generation,
client=gcs_client,
)
except google.api_core.exceptions.PreconditionFailed as err:
raise exceptions.BacklogException(
f"The lock at gs://{lock_blob.bucket.name}/{lock_blob.name} "
f"was changed by another process.") from err
def test_blob():
"""Provides a pre-existing blob in the test bucket."""
bucket = storage.Client().bucket(BUCKET)
blob_name = "test_blob_{}".format(uuid.uuid4().hex)
blob = Blob(
blob_name,
bucket,
encryption_key=TEST_ENCRYPTION_KEY_DECODED,
)
content = "Hello, is it me you're looking for?"
blob.upload_from_string(content)
yield blob.name, content
# To delete an encrypted blob, you have to provide the same key
# used for the blob. When you provide a wrong key, you'll get
# NotFound.
try:
# Clean up for the case that the rotation didn't occur.
blob.delete()
except NotFound as e:
# For the case that the rotation succeeded.
print("Ignoring 404, detail: {}".format(e))
blob = Blob(blob_name,
bucket,
encryption_key=TEST_ENCRYPTION_KEY_2_DECODED)
blob.delete()
def upload(csvdata, bucketname, blobname):
client = storage.Client()
bucket = client.get_bucket(bucketname)
blob = Blob(blobname, bucket)
blob.upload_from_string(csvdata)
gcslocation = 'gs://{}/{}'.format(bucketname, blobname)
logging.info('Uploaded {} ...'.format(gcslocation))
return gcslocation
def _set_data(self, **kwargs) -> dict:
df = kwargs.get('data_frame')
ts = time.time_ns()
blob_name_parts = os.path.splitext(self.__blob_name)
blob_name = blob_name_parts[0] + '_' + str(ts) + blob_name_parts[1]
blob = Blob(blob_name, self.__bucket)
blob.upload_from_string(df.to_csv(), self.__file_format)
return dict(record_count=df.shape[0])
def test_blob():
"""Provides a pre-existing blob in the test bucket."""
bucket = storage.Client().bucket(BUCKET)
blob = Blob('encryption_test_sigil',
bucket,
encryption_key=TEST_ENCRYPTION_KEY_DECODED)
content = 'Hello, is it me you\'re looking for?'
blob.upload_from_string(content)
return blob.name, content
def store_well(uwi):
bucketName = request.get_json()['bucket']
logging.info('Storing well {} to bucket {}.'.format(uwi, bucketName))
# Fetch entity with id
ds = datastore.Client(project=globalproject)
kind = wellKind
query = ds.query(kind=kind)
query.order = ['uwi']
result = get_fetch_results(query, uwi)
if len(result) == 0:
return make_response("Well not found", 404)
entity = result[0]
geolocation = {}
geolocation['longitude'] = entity['longitude']
geolocation['latitude'] = entity['latitude']
wellObj = {}
wellObj['md'] = entity['md']
wellObj['uwi'] = entity['uwi']
wellObj['name'] = entity['name']
wellObj['location'] = geolocation
wellObj['spuddate'] = entity['spuddate']
jsonObj = json.dumps(wellObj)
gcs = storage.Client(project=globalproject)
try:
# Check if the bucket exists
bucket = gcs.get_bucket(bucketName)
#store json to bucket
filename = str(id)
blob = Blob(filename, bucket)
try:
data = jsonObj.encode('utf-8')
blob.upload_from_string(data, content_type='text/plain')
logging.info("File " + filename + " stored in bucket " +
bucketName)
return make_response("Successfully stored in GCS", 200)
except:
return make_response('Error: Cannot store json object', 404)
except exceptions.NotFound:
return make_response(
'Error: Bucket {} does not exist.'.format(bucketName), 404)
except exceptions.BadRequest:
return make_response(
'Error: Invalid bucket name {}'.format(bucketName), 400)
except exceptions.Forbidden:
return make_response(
'Error: Forbidden, Access denied for bucket {}'.format(bucketName),
403)
def add(x):
time.sleep(1)
bucket = client.get_bucket('testdownload12')
blob = Blob("%s" % x, bucket)
x = x.encode("UTF8")
whoisdata = whois.whois(x)
whoisdata = json.dumps(whoisdata, indent=4, sort_keys=True, default=str)
blob.upload_from_string(whoisdata)
return x
def get_container(self):
try:
container = self.storage_client.get_bucket(self.CONTAINER)
# Test if the container is accessible
blob = Blob('AccessTestByServiceFabrikPythonLibrary', container)
blob.upload_from_string(
'Sample Message for AccessTestByServiceFabrikPythonLibrary', content_type='text/plain')
blob.delete()
return container
except Exception as error:
self.logger.error('[GCP] [STORAGE] ERROR: Unable to find or access container {}.\n{}'.format(
self.CONTAINER, error))
return None
def download_to_file(client, to_delete):
# [START download_to_file]
from google.cloud.storage import Blob
client = storage.Client(project='my-project')
bucket = client.get_bucket('my-bucket')
encryption_key = 'c7f32af42e45e85b9848a6a14dd2a8f6'
blob = Blob('secure-data', bucket, encryption_key=encryption_key)
blob.upload_from_string('my secret message.')
with open('/tmp/my-secure-file', 'wb') as file_obj:
blob.download_to_file(file_obj)
# [END download_to_file]
to_delete.append(blob)
def download_to_file(to_delete):
# [START download_to_file]
from google.cloud.storage import Blob
client = storage.Client(project="my-project")
bucket = client.get_bucket("my-bucket")
encryption_key = "c7f32af42e45e85b9848a6a14dd2a8f6"
blob = Blob("secure-data", bucket, encryption_key=encryption_key)
blob.upload_from_string("my secret message.")
with open("/tmp/my-secure-file", "wb") as file_obj:
client.download_to_file(blob, file_obj)
# [END download_to_file]
to_delete.append(blob)
def upload_blob_string(bucket_name, csvString, destination_blob_name):
client = storage.Client()
bucket = client.get_bucket(bucket_name)
blob = Blob(destination_blob_name, bucket)
return blob.upload_from_string(
data=csvString,
content_type='text/csv')
def upload_data(self, data, dest, contentType=None):
if self.use_gcs:
args = {}
if not contentType is None:
args = {'content_type': contentType}
blob = Blob(dest, self.gcs_bucket)
blob.upload_from_string(data, **args)
else:
args = {
'Body': data,
'Bucket': self.bucket,
'Key': dest,
}
if not contentType is None:
args['ContentType'] = contentType
self.s3.put_object(**args)
def store_file_to_gcs(self, bucket_name, datastr, filename):
if self.check_bucket(bucket_name) == False:
self.create_bucket(bucket_name)
print 'bucket created!'
bucket = self.gcs.get_bucket(bucket_name)
blob = Blob(filename, bucket)
try:
blob.upload_from_string(datastr)
print 'blob stored ok'
return True
except Exception:
print('Error: Cannot upload the file {}'.format(filename))
return False
def store_file_to_gcs(self, bucket_name, filename, content):
if self.check_bucket(bucket_name):
try:
print "content: ", content
print "inside store_file_to_gcs"
bucket = self.gcs.get_bucket(bucket_name)
print bucket
blob = Blob(filename, bucket)
print "blob", blob
blob.upload_from_string(content, content_type='application/json')
print "upload successful"
return True
except IOError as e:
print e
#print ('Error: Cannot find the file {}'.format(filename))
return False
def handle_bq_lock(gcs_client: storage.Client,
lock_blob: storage.Blob,
next_job_id: Optional[str],
table: bigquery.TableReference,
retry_attempt_cnt: Optional[int] = None):
"""Reclaim the lock blob for the new job id (in-place) or delete the lock
blob if next_job_id is None."""
try:
if next_job_id:
lock_blob_contents = json.dumps(
dict(job_id=next_job_id,
table=table.to_api_repr(),
retry_attempt_cnt=retry_attempt_cnt))
logging.log_with_table(
table,
f"Writing the following content to lock_blob {lock_blob.name}:"
f" {dict(job_id=next_job_id, table=table.to_api_repr(), retry_attempt_cnt=retry_attempt_cnt)}"
)
if lock_blob.exists(client=gcs_client):
lock_blob.upload_from_string(
lock_blob_contents,
if_generation_match=lock_blob.generation,
client=gcs_client)
else: # This happens when submitting the first job in the backlog
lock_blob.upload_from_string(
lock_blob_contents,
if_generation_match=0, # noqa: E126
client=gcs_client)
else:
logging.log_with_table(
table, "releasing lock at: "
f"gs://{lock_blob.bucket.name}/{lock_blob.name}")
lock_blob.delete(
if_generation_match=lock_blob.generation,
client=gcs_client,
)
except (google.api_core.exceptions.PreconditionFailed,
google.api_core.exceptions.NotFound) as err:
if isinstance(err, google.api_core.exceptions.PreconditionFailed):
raise exceptions.BacklogException(
f"The lock at gs://{lock_blob.bucket.name}/{lock_blob.name} "
f"was changed by another process.") from err
logging.log_with_table(
table,
"Tried deleting a lock blob that was either already deleted "
"or never existed.")
def master(event, context):
try:
pubsub_message = base64.b64decode(event['data']).decode('utf-8')
msg = json.loads(pubsub_message)
logging.debug("Image: " + msg['id'])
##1. Get image details
response = requests.get("http://interview.agileengine.com/images/" + msg['id'],headers={'Authorization': 'Bearer ' + msg['token']})
image_detail = json.loads(response.text)
##2. Save metadata into Firestore
db = firestore.Client()
data_tag = []
tag_values = image_detail['tags'].strip().split(" ")
for tag in tag_values:
data_tag.append({ u'tag': tag })
data = {
u'id': msg['id'],
u'author': image_detail['author'],
u'camera': image_detail['camera'],
u'tags': data_tag,
u'full_picture': image_detail['full_picture'],
u'cropped_picture': image_detail['cropped_picture'],
u'updated': datetime.today()
}
db.collection(u'Photos').document(msg['id']).set(data)
##3. Download image from URL
datatoupload = requests.get(image_detail['full_picture'], stream = True)
datatoupload.raw.decode_content = True
##4. Save image into Google Bucket
file_name = msg['id'] + ".jpg"
_google_client = storage.Client()
bucket = _google_client.get_bucket('photos_cache')
blob = Blob(file_name, bucket)
blob.upload_from_string(datatoupload.raw.data)
except Exception as e:
logging.error(str(e))
def deidentify_with_mask(data,done):
# Convert the project id into a full resource id.
parent = dlp.project_path(PROJECT_ID)
# Construct inspect configuration dictionary
inspect_config = {
'info_types': [{'name': info_type} for info_type in INFO_TYPES]
}
# Construct deidentify configuration dictionary
deidentify_config = {
'info_type_transformations': {
'transformations': [
{
'primitive_transformation': {
'character_mask_config': {
'masking_character': 'X',
'number_to_mask': 0
}
}
}
]
}
}
storage_client = storage.Client()
bucket = storage_client.get_bucket(SENSITIVE_BUCKET)
blobs = bucket.list_blobs()
for blob in blobs:
gcs_file = blob.download_as_string()
#contents = gcs_file.readline()
item = {'value': gcs_file}
# Call the API
response = dlp.deidentify_content(
parent, inspect_config=inspect_config,
deidentify_config=deidentify_config, item=item)
masked_item = response.item.value
destination_bucket = storage_client.get_bucket(MASKED_BUCKET)
masked_blob = Blob(blob.name,destination_bucket)
masked_blob.upload_from_string(masked_item)
def upload():
"""Process the uploaded file and upload it to Google Cloud Storage."""
uploaded_file = request.files['file']
bucket_name = other_bucket
size = len(uploaded_file.read())
extension = os.path.splitext(uploaded_file.filename)[1]
jpg = ".jpg"
# if size > 2000:
# return "Cannot allow file larger than 20 kb"
if extension == jpg:
bucket_name = pic_bucket
client = storage.Client()
bucket = client.get_bucket(bucket_name)
#encoded_key = base64.b64encode(key).decode('utf-8')
#encryption_key = base64.b64decode(encoded_key)
blob = Blob(uploaded_file.filename, bucket)
blob.upload_from_string(uploaded_file.read())
#blob.make_public()
# if blob.exists():
# return "already exists"
# else:
# return "doesnt exist"
# blob.upload_from_file(uploaded_file)
# #Print filesize
# object_list=[]
# total_size = 0
# blobs = bucket.list_blobs()
# for blob in blobs:
# object_list.append(blob.name+' '+str(blob.size))
# object_list.append("<br>")
# total_size = total_size + blob.size
# str1 = ''.join(object_list)
# return str1 + 'Total Size:' +str(total_size) + '<br>' + 'Time Created' + blob.metadata
return str(blob.time_created)
def sendBucket(id):
try:
ds = datastore.Client(project='hackathon-team-011')
key = ds.key('capitals', int(id))
entity = ds.get(ds.key('capitals', int(id)))
if entity is None:
response = {'code': 404, 'message': 'Capital not found'}
return jsonify(response), 404
gcs = storage.Client(project='hackathon-team-011')
obj = request.get_json()
bucketname = obj['bucket']
bucket = gcs.get_bucket(bucketname)
filename = str(id)
blob = Blob(filename, bucket)
#fs = open(filename, 'w')
#fs.write(json.dumps(parse_capital(entity)))
#fs.close()
#fs = open(filename, 'r')
blob.upload_from_string(json.dumps(parse_capital(entity)))
#fs.close()
response = {
'code': 200,
'message':
'Capital successfully stored in GCS in file: ' + filename
}
return jsonify(response), 200
except exceptions.NotFound:
response = {
'code': 404,
'message': 'Error: Bucket {} does not exists.'.format(bucketname)
}
return jsonify(response), 404
except Exception as e:
response = {'code': 0, 'message': 'Unexpected error' + e.message}
return jsonify(response)
def store(self, bucket_name, id, entity):
bucket_exists = self.check_bucket(bucket_name)
if bucket_exists is not None and not bucket_exists:
try:
print('creating bucket {}'.format(bucket_name))
bucket = self.gcs.create_bucket(bucket_name)
blob = Blob(id, bucket)
js = json.dumps(entity)
resp = Response(js, status=200, mimetype='application/json')
blob.upload_from_string(entity)
except Exception as e:
print "Error: Create bucket Exception"
print e
return None
else:
bucket = self.gcs.get_bucket(bucket_name)
blob = Blob(id, bucket)
js = json.dumps(entity)
resp = Response(js, status=200, mimetype='application/json')
blob.upload_from_string(entity)
return 200
def diarize(data, context):
speech_file = data['name']
bucket = data['bucket']
print('Bucket {}'.format(bucket))
print('File {}'.format(speech_file))
filename_uri = "gs://" + bucket + "/" + speech_file
print('File name uri {}'.format(filename_uri))
dest_file = speech_file + ".txt"
audio = speech.types.RecognitionAudio(uri=filename_uri)
config = speech.types.RecognitionConfig(
encoding=speech.enums.RecognitionConfig.AudioEncoding.LINEAR16,
sample_rate_hertz=8000,
language_code='en-US',
enable_speaker_diarization=True,
diarization_speaker_count=2)
operation = client.long_running_recognize(config, audio)
print('Waiting for operation to complete...')
# The transcript within each result is separate and sequential per result.
# However, the words list within an alternative includes all the words
# from all the results thus far. Thus, to get all the words with speaker
# tags, you only have to take the words list from the last result:
#print(response.results)
response = operation.result(timeout=300)
result = response.results[-1]
words_info = result.alternatives[0].words
writestring = ''
for word_info in words_info:
writestring += "Word: {} Speaker Tag: {}\n".format(
word_info.word, word_info.speaker_tag)
storage_client = storage.Client()
bucket = storage_client.get_bucket(bucket)
blob = Blob(dest_file, bucket)
blob.upload_from_string(writestring)
def upload_blob_string(bucket_name, data_string, destination_blob_name,
metadata, content_type):
logging.debug(
'upload_blob_string(...) \n\tbucket_name={}\n\tlen data_string={}\n\t'
'destination_blob_name={}\n\tmetadata={}\n\tcontent_type={}'.format(
bucket_name, len(data_string), destination_blob_name, metadata,
content_type))
client = storage.Client()
bucket = client.get_bucket(bucket_name)
blob = Blob(destination_blob_name, bucket)
if blob.metadata is None:
blob.metadata = metadata
else:
blob.metadata.update(metadata)
return blob.upload_from_string(data=data_string, content_type=content_type)
def execute(request):
# Check Payload
if not request.form or not 'external_ref' in request.form:
return ('Bad request: External Ref is required', 400)
if not request.form or not 'external_key' in request.form:
return ('Bad request: External Key is required', 400)
if not request.files or not 'file' in request.files:
return ('Bad request: File is required', 400)
# Get external ref
external_ref = request.form.get('external_ref')
# Get external key
external_key = request.form.get('external_key')
# External ref uuid name
external_ref_uuid_name = 'extref=%s%s' % (slugify.slugify(external_ref), external_key)
# Generate UUID external ref
external_ref_uuid = uuid.uuid5(uuid.NAMESPACE_X500, external_ref_uuid_name)
# Get external ref uuid str
external_ref_uuid_str = str(uuid.uuid5(uuid.NAMESPACE_X500, external_ref_uuid_name))
# Get unique key external ref
external_ref_key = hashlib.md5(external_ref_uuid.bytes).hexdigest()
try: # Get/Create bucket
client_storage = storage.Client()
bucket = client_storage.create_bucket(BUCKET_NAME)
except google.api_core.exceptions.Conflict: # Bucket already exists
bucket = client_storage.get_bucket(BUCKET_NAME)
except Exception as e: # Any exception
return (u'Error: %s' % e, 500)
# Bucket file uuid name
encryption_key_name = 'extrefkey=%s' % external_ref_key
# Generate UUID Bucket file
encryption_key_uuid = uuid.uuid5(uuid.NAMESPACE_X500, encryption_key_name)
# Generate encryption key Bucket file
encryption_key = hashlib.md5(encryption_key_uuid.bytes).hexdigest()
try: # Get file
file = request.files.get('file')
# Generate path file
path_file = "%s/%s/%s" % (str(external_ref_uuid), str(encryption_key_uuid), file.filename)
# Create encripty Blob
blob = Blob(name=path_file, bucket=bucket, encryption_key=encryption_key)
except Exception as e:
return (u'Error: %s' % e, 500)
try: # Read and upload file into bucket
buff = file.read()
blob.upload_from_string(buff, content_type=file.content_type)
except Exception as e: # Any exception
return (e, 500)
# Generate File hash
file_hash = hashlib.md5(buff).hexdigest()
try: # Create entity
client_datastore = datastore.Client()
# Generate Datastore Key
item_key = client_datastore.key(DS_KIND, "%s-%s" % (external_ref_key, file_hash))
# Entity
item = datastore.Entity(key=item_key,) # Insert user key
item['external_ref_key'] = external_ref_key
item['file_hash'] = file_hash
item['file_path'] = path_file
item['file_content_type'] = file.content_type
client_datastore.put(item)
except Exception as e: # Any exception
return (u'Error: %s' % e, 500)
# Data return
data = json.dumps({'filename': file.filename,'hash' : file_hash})
# Response
return Response(data, mimetype='application/json')
def upload_document(document, uid):
"""
Uploads given document to given user's folder
If a file like "documentname.ext" already exists,
find next best name as so: "documentname_({i})" where {i} is a number between
1 and the number of similarly-named files + 1
Args:
document - File blob given by user
uid - firestore user's document id
"""
try:
new_file = None
flags = re.U | re.I
file_no_ext = re.sub(r"\.[a-z]+$", "", document.filename, flags=flags)
file_ext = re.search(r"\.[a-z]+$", document.filename, flags).group(0)
# Lists current files in the database
blobs = storage_client.list_blobs(bucket,
prefix=get_cloud_folder(uid) +
file_no_ext)
# Grabs all files with the same basic file name
files = [
re.search(r"[^\\\/:*?\"<>]+\.[a-z]+$", blob.name, flags).group(0)
for blob in blobs
]
next_file_name = None
# If no similarly-named files or the first one is not an exact match
if len(files) == 0 or files[0] != document.filename:
# New file keeps its original name
new_file = Blob(
get_cloud_folder(uid) + document.filename,
bucket,
encryption_key=encryption_key,
)
if new_file == None:
for i, blob in enumerate(files):
# Tries to find an open similar file name between 1 and the number of files
next_file_name = str.format("{}_({}){}", file_no_ext, i + 1,
file_ext)
if next_file_name not in files:
print(next_file_name, blob)
new_file = Blob(
get_cloud_folder(uid) + next_file_name,
bucket,
encryption_key=encryption_key,
)
break
# If no open filename
if new_file == None:
# New file name is the number of files + 1
next_file_name = str.format("{}_({}){}", file_no_ext, i + 1,
file_ext)
new_file = Blob(
get_cloud_folder(uid) + next_file_name,
bucket,
encryption_key=encryption_key,
)
# Creates file from the given blob and uploads it to Google Cloud Storage
new_file.upload_from_string(document.read(),
content_type=document.content_type)
except Exception as e:
print(e)
def _write_string(self, bucket_name, file_name, data):
log.info(f"Storing data in {bucket_name}:{file_name}")
blob = Blob(file_name, self.get_or_create_bucket(bucket_name))
blob.upload_from_string(data)
def __create_new_timestamp_blob(self):
blob = Blob(self.last_timestamp_filename, self.bucket)
blob.upload_from_string('0')
return blob
def sync_box_to_gcs(box: BoxClient, bucket: Bucket,
cache: dict) -> List[Future]:
"""Sync Box account files to a GCS bucket.
For versioned Box files, the latest version will always be synced back to the GCS bucket.
Non-current versions will not be deliberately preserved, though syncing to a versioned
bucket will have this effect.
Arguments:
box {BoxClient} -- [description]
bucket {Bucket} -- [description]
cache {dict} -- A dictionary that will opportunistically be filled with Box item paths/objects.
Returns:
List[Future] -- [description]
"""
# constuct an executor for copy tasks
executor = ThreadPoolExecutor(max_workers=cpu_count())
futures = []
# sync box files back to GCS
for path, item in box_walk(box.root_folder()):
LOG.debug("Box directory walk found: {}".format(path))
# cache the Box item in module scope
cache[path] = item
# get the blob to overwrite, or make a new one
blob_exists = True
blob = bucket.get_blob(path)
if not blob:
blob_exists = False
blob = Blob(path, bucket)
# branch depending on whether file or folder
if item.type == "folder":
if not blob_exists:
blob.metadata = {
# Not an important value.
BOX_MTIME_KEY: datetime.now().isoformat()
}
# create directory marker, used by UI and FUSE
LOG.info("Creating directory marker in GCS: {}".format(
blob.name))
blob.upload_from_string(b'')
elif item.type == "file":
box_file = box.file(item.id)
box_mtime = box_file.get().modified_at
if should_copy_box_to_gcs(box_file, box_mtime, blob, blob_exists):
LOG.info(
"Box file {} is not found in GCS or updated since last sync. Copying to {}."
.format(item.name, blob.name))
blob.metadata = {
BOX_MTIME_KEY: box_mtime
} # This change will "follow" the upload into GCS
temp_file = BytesIO()
reader = box_file.download_to
writer = blob.upload_from_file
future = executor.submit(concurrent_upload, reader, writer,
temp_file)
futures.append(future)
else:
LOG.info("Ignoring item of type {}".format(item.type))
return futures
def _create_folder_file():
folder_key = path.join(root, '{0}_$folder$'.format(folder_name))
blob = Blob(folder_key, gcp_bucket)
if not blob.exists():
blob.upload_from_string(data='')