def main(): print 'Bucket List:' for bucket in storage.buckets(): print bucket.name print '' print 'Information about datastudio-test bucket:' bucket = storage.bucket('datastudio-test') bucket_md = bucket.metadata print 'Name : ' + bucket_md.name print 'ETag : ' + bucket_md.etag print 'Created: ' + str(bucket_md.created_on) print '' print 'datastudio-not-existing exists?' print storage.buckets().contains('datastudio-non-existing-bucket') print '' print 'Item List:' for item in bucket.items(): print item.key + ':' + item.metadata().content_type print '' print 'Filtered Item List:' for item in bucket.items(prefix='folder1/', delimiter='/'): print item.key + ':' + item.metadata().content_type
def main(): bucket = storage.bucket('datastudio-test') items = bucket.items() print 'datastudio-test/test.txt exists? ' + str(items.contains('test.txt')) item = bucket.item('test.txt') item_md = item.metadata print 'Name : ' + item_md.name print 'ETag : ' + item_md.etag print 'Updated: ' + str(item_md.updated_on) print 'Content: ' + item_md.content_type print 'Size : ' + str(item_md.size) print item._info print 'Read Item content:' text = item.read_from() print text print '' print 'Write Item content:' text = text + '\n' + str(dt.datetime.utcnow()) item.write_to(text, 'text/plain') print '' print 'datastudio-test/non-existing-file exists? ' + str( items.contains('non-existing-file')) print 'Moving folder1/test3.txt to foo.txt' print 'folder1/test3.txt exists? ' + str( items.contains('folder1/test3.txt')) item2 = bucket.item('folder1/test3.txt') item3 = item2.copy_to('foo.txt') print 'foo.txt exists? ' + str(items.contains('foo.txt')) item2.delete() print 'folder1/test3.txt exists? ' + str( items.contains('folder1/test3.txt')) item3.copy_to('folder1/test3.txt') print 'folder1/test3.txt exists? ' + str( items.contains('folder1/test3.txt'))
def main(): bucket = storage.bucket('datastudio-test') items = bucket.items() print 'datastudio-test/test.txt exists? ' + str(items.contains('test.txt')) item = bucket.item('test.txt') item_md = item.metadata print 'Name : ' + item_md.name print 'ETag : ' + item_md.etag print 'Updated: ' + str(item_md.updated_on) print 'Content: ' + item_md.content_type print 'Size : ' + str(item_md.size) print item._info print 'Read Item content:' text = item.read_from() print text print '' print 'Write Item content:' text = text + '\n' + str(dt.datetime.utcnow()) item.write_to(text, 'text/plain') print '' print 'datastudio-test/non-existing-file exists? ' + str(items.contains('non-existing-file')) print 'Moving folder1/test3.txt to foo.txt' print 'folder1/test3.txt exists? ' + str(items.contains('folder1/test3.txt')) item2 = bucket.item('folder1/test3.txt') item3 = item2.copy_to('foo.txt') print 'foo.txt exists? ' + str(items.contains('foo.txt')) item2.delete() print 'folder1/test3.txt exists? ' + str(items.contains('folder1/test3.txt')) item3.copy_to('folder1/test3.txt') print 'folder1/test3.txt exists? ' + str(items.contains('folder1/test3.txt'))
def ipynb_to_code(bucket, notebook): import gcp import gcp.storage as storage bucket = storage.bucket('cloud-androidwear-notebooks') notebook_json = bucket.item("App Metrics.ipynb").read_from() import json as json_parser notebook = json_parser.loads(notebook_json) cells = notebook["worksheets"][0]["cells"] code_inputs = [cell["input"] for cell in cells if cell["cell_type"] == "code"] # Handle bq magic import re # output the bq_sql magic code def muggle_bq_sql(symbol, lines): return [symbol + ' = gcp.bigquery.query(gcp.bigquery.sql("""\n'] + lines + ['""", **locals()))']; # TODO: this would be a good place to handle all the magic def muggle(lines): # if we've got nothing, bail if lines == []: return lines # if we've got some bg_sql magic, handle it regex = re.compile("^\s*%%bq_sql\s*(?P<symbol>\S*)\s*$", re.IGNORECASE) match = regex.match(lines[0]) if match: # if we've got bq_sql w/o a symbol, do nothing symbol = match.groups("symbol")[0] if symbol == "": return [] # if we've got a symbol, muggle the bq_sql code else: return muggle_bq_sql(symbol, lines[1:]) # otherwise, let it through else: return lines muggle_code_inputs = map(muggle, code_inputs) flat_code = [item for sublist in muggle_code_inputs for item in sublist] imports = ["gcp.bigquery"] code = '\n'.join(["import " + i for i in imports]) + '\n' + '\n'.join(flat_code) return code