def upload(filename, bucket): client = storage.get_bucket( bucket, "tagtooadex2", "*****@*****.**", "tagtooadex2-9e2928ac0acf.p12" ) for i in os.listdir('.'): if '%s.' % filename in i and '.gz' not in i: print 'start upload', i, bucket ipath = './%s' % i os.system('gzip %s' % ipath) client.upload_file(ipath, ipath) os.remove(ipath)
#!/usr/bin/python import jinja2 import csv import logging import codecs from collections import defaultdict from gcloud import storage from jinja2 import Template template = Template(open('sponsor_page.html').read()) bucket = storage.get_bucket("pycon-apac-2015", "living-bio") def upload_image(filename): try: key = bucket.upload_file(filename) key.make_public() return key.public_url except: return 'http://storage.googleapis.com/pycon-apac-2015/%s' % filename def import_application(): level_orders = ["Gold", "Silver", "Bronze", "Media", "Other"] sponsor_categories = defaultdict(list) with open('application.csv') as ifile: ireader = csv.DictReader(ifile) for row in ireader: if not row['Sponsor Package']: continue
def _callFUT(self, *args, **kw): from gcloud.storage import get_bucket return get_bucket(*args, **kw)
i += step #-------------------------------------- # set default bucket #-------------------------------------- storage.set_default_bucket("isb-cgc") storage_conn = storage.get_connection() storage.set_default_connection(storage_conn) all_elements = {} df2 = pd.DataFrame() #-------------------------------------- # get the bucket contents #-------------------------------------- bucket = storage.get_bucket('ptone-experiments') for k in bucket.list_blobs(prefix="working-files/clinical_metadata/"): if 'counts.txt' in k.name: disease_type = k.name.split("/")[2].split(".")[0] data = StringIO() k.download_to_file(data) data.seek(0) df = pd.read_csv(data, sep="\t") df['disease_type'] = disease_type if df2.empty:
def _callFUT(self, *args, **kw): from gcloud.storage import get_bucket return get_bucket(*args, **kw)
#!/usr/bin/python import jinja2 import csv import logging import codecs from collections import defaultdict from gcloud import storage from jinja2 import Template template = Template(open('sponsor_page.html').read()) bucket = storage.get_bucket( "pycon-apac-2015", "living-bio" ) def upload_image(filename): try: key = bucket.upload_file(filename) key.make_public() return key.public_url except: return 'http://storage.googleapis.com/pycon-apac-2015/%s' % filename def import_application(): level_orders = ["Platinum", "Gold", "Silver", "Bronze", "Media", "Other"] sponsor_categories = defaultdict(list) with open('application.csv') as ifile: ireader = csv.DictReader(ifile)
def _get_bucket(): try: return storage.get_bucket(BUCKET_NAME) except NotFound: return storage.create_bucket(BUCKET_NAME)
#-------------------------------------- # set default bucket #-------------------------------------- storage.set_default_bucket("isb-cgc") storage_conn = storage.get_connection() storage.set_default_connection(storage_conn) all_elements = {} df2 = pd.DataFrame() #-------------------------------------- # get the bucket contents #-------------------------------------- bucket = storage.get_bucket('ptone-experiments') for k in bucket.list_blobs(prefix="working-files/clinical_metadata/"): if 'counts.txt' in k.name: disease_type = k.name.split("/")[2].split(".")[0] data = StringIO() k.download_to_file(data) data.seek(0) df = pd.read_csv(data, sep="\t") df['disease_type'] = disease_type if df2.empty:
def __init__(self, options): self.bucket = storage.get_bucket('just_a_test') self.bucket.make_public(recursive=True, future=True)
#-------------------------------------- # set default bucket #-------------------------------------- storage.set_default_bucket("isb-cgc") storage_conn = storage.get_connection() storage.set_default_connection(storage_conn) all_elements = {} #-------------------------------------- # get the bucket contents #-------------------------------------- bucket = storage.get_bucket('isb-cgc-open') for k in bucket.list_blobs(prefix="tcga/"): if '.xml' in k.name and 'clinical' in k.name: print k.name disease_type = k.name.split("/")[1] maf_data = StringIO() k.download_to_file(maf_data) maf_data.seek(0) tree = etree.parse(maf_data) root = tree.getroot() #this is the root; we can use it to find elements blank_elements = re.compile("^\\n\s*$")