def __attempt_upload(file_path, bucket_name, key_name, log):
time.sleep(backoff)
bucket = get_bucket(bucket_name)
if key_name in bucket:
raise ValueError('found %s in %s' % (key_name, bucket_name))
key = boto.gs.key.Key(bucket, key_name)
try:
key.set_contents_from_filename(file_path)
finally:
key.close()
def __attempt_download(file_path, bucket_name, key_name, log):
time.sleep(backoff)
bucket = __get_bucket(bucket_name)
if key_name not in bucket:
raise ValueError('did not find %s in %s' % (key_name, bucket_name))
key = boto.gs.key.Key(bucket, key_name)
try:
key.get_contents_to_filename(file_path)
finally:
key.close()
log.info('successfully downloaded %s' % key_name)
def deleteGCSobj ( gcsPath ):
( bucketName, oName ) = splitGCSpath ( gcsPath )
## print " --> ", bucketName, oName
connection = boto.gs.connection.GSConnection()
bucket = connection.get_bucket(bucketName)
if oName not in bucket:
raise ValueError('did not find %s in %s' % (oName, bucketName))
key = boto.gs.key.Key(bucket, oName)
try:
key.delete()
finally:
key.close()
connection.close()
def deleteGCSobj ( gcsPath ):
( bucketName, oName ) = splitGCSpath ( gcsPath )
## print " --> ", bucketName, oName
connection = boto.gs.connection.GSConnection()
bucket = connection.get_bucket(bucketName)
if oName not in bucket:
raise ValueError('did not find %s in %s' % (oName, bucketName))
key = boto.gs.key.Key(bucket, oName)
try:
key.delete()
finally:
key.close()
connection.close()
def __attempt_upload(file_path, bucket_name, key_name, log):
time.sleep(backoff)
if bucket_name in name2bucket:
bucket = name2bucket[bucket_name]
else:
with lock:
if bucket_name in name2bucket:
bucket = name2bucket[bucket_name]
else:
bucket = connection.get_bucket(bucket_name)
name2bucket[bucket_name] = bucket
if key_name in bucket:
raise ValueError('found %s in %s' % (key_name, bucket_name))
key = boto.gs.key.Key(bucket, key_name)
try:
key.set_contents_from_filename(file_path)
finally:
key.close()
def __attempt_upload(file_path, bucket_name, key_name, log):
time.sleep(backoff)
if bucket_name in name2bucket:
bucket = name2bucket[bucket_name]
else:
with lock:
if bucket_name in name2bucket:
bucket = name2bucket[bucket_name]
else:
bucket = connection.get_bucket(bucket_name)
name2bucket[bucket_name] = bucket
if key_name in bucket:
raise ValueError('found %s in %s' % (key_name, bucket_name))
key = boto.gs.key.Key(bucket, key_name)
try:
key.set_contents_from_filename(file_path)
finally:
key.close()
def __attempt_upload(file_path, bucket_name, key_name, log):
time.sleep(backoff)
bucket = __get_bucket(bucket_name)
if key_name in bucket:
raise ValueError('found %s already uploaded in %s' %
(key_name, bucket_name))
key = boto.gs.key.Key(bucket, key_name)
try:
if (file_path.endswith('txt')):
# make sure can be loaded as ascii!!!
with open(file_path) as infile:
contents = infile.read()
key.set_contents_from_string(
str(contents.encode('ascii', 'replace')))
else:
key.set_contents_from_filename(file_path)
finally:
key.close()