def __init__(self, oauth_credentials=None, descriptor='', http_=None):
authenticate_kwargs = gcp.get_authenticate_kwargs(oauth_credentials, http_)
if descriptor:
self.client = discovery.build_from_document(descriptor, **authenticate_kwargs)
else:
self.client = discovery.build('bigquery', 'v2', **authenticate_kwargs)
def __init__(self, oauth_credentials=None, descriptor='', http_=None):
authenticate_kwargs = gcp.get_authenticate_kwargs(oauth_credentials, http_)
if descriptor:
self.client = discovery.build_from_document(descriptor, **authenticate_kwargs)
else:
self.client = discovery.build('bigquery', 'v2', cache_discovery=False, **authenticate_kwargs)
def __init__(self, oauth_credentials=None, descriptor='', http_=None,
chunksize=CHUNKSIZE):
self.chunksize = chunksize
authenticate_kwargs = gcp.get_authenticate_kwargs(oauth_credentials, http_)
if descriptor:
self.client = discovery.build_from_document(descriptor, **authenticate_kwargs)
else:
self.client = discovery.build('storage', 'v1', **authenticate_kwargs)
def __initialise_client(self):
authenticate_kwargs = gcp.get_authenticate_kwargs(
self.oauth_credentials, self.http_)
if self.descriptor:
self.client = discovery.build_from_document(
self.descriptor, **authenticate_kwargs)
else:
self.client = discovery.build('bigquery',
'v2',
cache_discovery=False,
**authenticate_kwargs)
def __init__(self, oauth_credentials=None, descriptor='', http_=None,
chunksize=CHUNKSIZE, **discovery_build_kwargs):
self.chunksize = chunksize
authenticate_kwargs = gcp.get_authenticate_kwargs(oauth_credentials, http_)
build_kwargs = authenticate_kwargs.copy()
build_kwargs.update(discovery_build_kwargs)
if descriptor:
self.client = discovery.build_from_document(descriptor, **build_kwargs)
else:
build_kwargs.setdefault('cache_discovery', False)
self.client = discovery.build('storage', 'v1', **build_kwargs)
def __init__(self, oauth_credentials=None, descriptor='', http_=None,
chunksize=CHUNKSIZE, **discovery_build_kwargs):
self.chunksize = chunksize
authenticate_kwargs = gcp.get_authenticate_kwargs(oauth_credentials, http_)
build_kwargs = authenticate_kwargs.copy()
build_kwargs.update(discovery_build_kwargs)
if descriptor:
self.client = discovery.build_from_document(descriptor, **build_kwargs)
else:
build_kwargs.setdefault('cache_discovery', False)
self.client = discovery.build('storage', 'v1', **build_kwargs)
def __init__(self,
oauth_credentials=None,
descriptor='',
http_=None,
chunksize=CHUNKSIZE):
self.chunksize = chunksize
authenticate_kwargs = gcp.get_authenticate_kwargs(
oauth_credentials, http_)
if descriptor:
self.client = discovery.build_from_document(
descriptor, **authenticate_kwargs)
else:
self.client = discovery.build('storage', 'v1',
**authenticate_kwargs)
import logging
import luigi
from luigi.contrib import gcp
logger = logging.getLogger('luigi-interface')
_dataproc_client = None
try:
import oauth2client.client
from googleapiclient import discovery
from googleapiclient.errors import HttpError
DEFAULT_CREDENTIALS = oauth2client.client.GoogleCredentials.get_application_default(
)
authenticate_kwargs = gcp.get_authenticate_kwargs(DEFAULT_CREDENTIALS)
_dataproc_client = discovery.build('dataproc', 'v1', **authenticate_kwargs)
except ImportError:
logger.warning(
"Loading Dataproc module without the python packages googleapiclient & oauth2client. \
This will crash at runtime if Dataproc functionality is used.")
def get_dataproc_client():
return _dataproc_client
def set_dataproc_client(client):
global _dataproc_client
_dataproc_client = client
import time
import logging
import luigi
from luigi.contrib import gcp
logger = logging.getLogger('luigi-interface')
_dataproc_client = None
try:
import oauth2client.client
from googleapiclient import discovery
from googleapiclient.errors import HttpError
DEFAULT_CREDENTIALS = oauth2client.client.GoogleCredentials.get_application_default()
authenticate_kwargs = gcp.get_authenticate_kwargs(DEFAULT_CREDENTIALS)
_dataproc_client = discovery.build('dataproc', 'v1', **authenticate_kwargs)
except ImportError:
logger.warning("Loading Dataproc module without the python packages googleapiclient & oauth2client. \
This will crash at runtime if Dataproc functionality is used.")
def get_dataproc_client():
return _dataproc_client
def set_dataproc_client(client):
global _dataproc_client
_dataproc_client = client
