class SqoopClient(object):
STATUS_GOOD = ('FINE', 'ACCEPTABLE')
STATUS_BAD = ('UNACCEPTABLE', 'FAILURE_ON_SUBMIT')
def __init__(self, url, username, language='en'):
self._url = url
self._client = HttpClient(self._url, logger=LOG)
self._root = SqoopResource(self._client)
self._language = language
self._username = username
def __str__(self):
return "SqoopClient at %s" % self._url
@property
def url(self):
return self._url
@property
def headers(self):
return {
'Accept': 'application/json',
'Accept-Language': self._language,
'sqoop-user-name': self._username
}
def get_version(self):
return self._root.get('version', headers=self.headers)
def get_driver(self):
resp_dict = self._root.get('%s/driver' % API_VERSION,
headers=self.headers)
driver = Driver.from_dict(resp_dict)
return driver
def get_connectors(self):
resp_dict = self._root.get('%s/connectors' % API_VERSION,
headers=self.headers)
connectors = [
Connector.from_dict(connector_dict)
for connector_dict in resp_dict['connectors']
]
return connectors
def get_connector(self, connector_id):
resp_dict = self._root.get('%s/connector/%d/' %
(API_VERSION, connector_id),
headers=self.headers)
if resp_dict['connector']:
return Connector.from_dict(resp_dict['connector'])
return None
def get_links(self):
resp_dict = self._root.get('%s/links' % API_VERSION,
headers=self.headers)
links = [Link.from_dict(link_dict) for link_dict in resp_dict['links']]
return links
def get_link(self, link_id):
resp_dict = self._root.get('%s/link/%d/' % (API_VERSION, link_id),
headers=self.headers)
if resp_dict['link']:
return Link.from_dict(resp_dict['link'])
return None
def create_link(self, link):
link.creation_date = int(round(time.time() * 1000))
link.update_date = link.creation_date
link_dict = link.to_dict()
request_dict = {'link': link_dict}
resp = self._root.post('%s/link/' % API_VERSION,
data=json.dumps(request_dict),
headers=self.headers)
# Lame check that iterates to make sure we have an error
# Server responds with: {'validation-result': [{},{}]} or {'validation-result': [{KEY: ERROR},{KEY: ERROR}]}
for result in resp['validation-result']:
if result:
raise SqoopException.from_dicts(resp['validation-result'])
link.id = resp['id']
return link
def update_link(self, link):
if not link.link_config_values:
link.link_config_values = self.get_connectors()[0].link_config
link.updated = int(round(time.time() * 1000))
link_dict = link.to_dict()
request_dict = {'link': link_dict}
resp = self._root.put('%s/link/%d/' % (API_VERSION, link.id),
data=json.dumps(request_dict),
headers=self.headers)
# Lame check that iterates to make sure we have an error
# Server responds with: {'validation-result': [{},{}]} or {'validation-result': [{KEY: ERROR},{KEY: ERROR}]}
for result in resp['validation-result']:
if result:
raise SqoopException.from_dicts(resp['validation-result'])
return link
def delete_link(self, link):
resp = self._root.delete('%s/link/%d/' % (API_VERSION, link.id),
headers=self.headers)
return None
def get_jobs(self):
resp_dict = self._root.get('%s/jobs' % API_VERSION,
headers=self.headers)
jobs = [Job.from_dict(job_dict) for job_dict in resp_dict['jobs']]
return jobs
def get_job(self, job_id):
resp_dict = self._root.get('%s/job/%d/' % (API_VERSION, job_id),
headers=self.headers)
if resp_dict['job']:
return Job.from_dict(resp_dict['job'])
return None
def create_job(self, job):
if not job.from_config_values:
job.from_config_values = self.get_connectors(
)[0].job_config['FROM']
if not job.to_config_values:
job.to_config_values = self.get_connectors()[0].job_config['TO']
if not job.driver_config_values:
job.driver_config_values = self.get_driver().job_config
job.creation_date = int(round(time.time() * 1000))
job.update_date = job.creation_date
job_dict = job.to_dict()
request_dict = {'job': job_dict}
resp = self._root.post('%s/job/' % API_VERSION,
data=json.dumps(request_dict),
headers=self.headers)
if 'id' not in resp:
raise SqoopException.from_dicts(resp['validation-result'])
job.id = resp['id']
return job
def update_job(self, job):
if not job.from_config_values:
job.from_config_values = self.get_connectors(
)[0].job_config['FROM']
if not job.to_config_values:
job.to_config_values = self.get_connectors()[0].job_config['TO']
if not job.driver_config_values:
job.driver_config_values = self.get_driver().job_config
job.updated = int(round(time.time() * 1000))
job_dict = job.to_dict()
request_dict = {'job': job_dict}
resp = self._root.put('%s/job/%d/' % (API_VERSION, job.id),
data=json.dumps(request_dict),
headers=self.headers)
# Lame check that iterates to make sure we have an error
# Server responds with: {'validation-result': [{},{}]} or {'validation-result': [{KEY: ERROR},{KEY: ERROR}]}
for result in resp['validation-result']:
if result:
raise SqoopException.from_dicts(resp['validation-result'])
return job
def delete_job(self, job):
resp_dict = self._root.delete('%s/job/%s' % (API_VERSION, job.id),
headers=self.headers)
return None
def get_job_status(self, job):
resp_dict = self._root.get('%s/job/%d/status' % (API_VERSION, job.id),
headers=self.headers)
return Submission.from_dict(resp_dict['submission'])
def start_job(self, job):
resp_dict = self._root.put('%s/job/%d/start' % (API_VERSION, job.id),
headers=self.headers)
if resp_dict['submission']['status'] in SqoopClient.STATUS_BAD:
raise SqoopSubmissionException.from_dict(resp_dict['submission'])
return Submission.from_dict(resp_dict['submission'])
def stop_job(self, job):
resp_dict = self._root.put('%s/job/%d/stop' % (API_VERSION, job.id),
headers=self.headers)
return Submission.from_dict(resp_dict['submission'])
def get_submissions(self):
resp_dict = self._root.get('%s/submissions' % API_VERSION,
headers=self.headers)
submissions = [
Submission.from_dict(submission_dict)
for submission_dict in resp_dict['submissions']
]
return submissions
def set_user(self, user):
self._user = user
def set_language(self, language):
self._language = language
class SqoopClient(object):
"""
Sqoop client
"""
STATUS_GOOD = ('FINE', 'ACCEPTABLE')
STATUS_BAD = ('UNACCEPTABLE', 'FAILURE_ON_SUBMIT')
def __init__(self, url, username, language='en'):
self._url = url
self._client = HttpClient(self._url, logger=LOG)
self._root = SqoopResource(self._client)
self._language = language
self._username = username
def __str__(self):
return "SqoopClient at %s" % self._url
@property
def url(self):
return self._url
@property
def headers(self):
return {
'Accept': 'application/json',
'Accept-Language': self._language,
'sqoop-user-name': self._username
}
def get_version(self):
return self._root.get('version', headers=self.headers)
def get_framework(self):
resp_dict = self._root.get('%s/framework/all' % API_VERSION, headers=self.headers)
framework = Framework.from_dict(resp_dict)
return framework
def get_connectors(self):
resp_dict = self._root.get('%s/connector/all' % API_VERSION, headers=self.headers)
connectors = [ Connector.from_dict(connector_dict, resp_dict['resources-connector']) for connector_dict in resp_dict['all'] ]
return connectors
def get_connector(self, connector_id):
resp_dict = self._root.get('%s/connector/%d/' % (API_VERSION, connector_id), headers=self.headers)
if resp_dict['all']:
return Connector.from_dict(resp_dict['all'][0], resp_dict['resources-connector'])
return None
def get_connections(self):
resp_dict = self._root.get('%s/connection/all' % API_VERSION, headers=self.headers)
connections = [Connection.from_dict(conn_dict) for conn_dict in resp_dict['all']]
return connections
def get_connection(self, connection_id):
resp_dict = self._root.get('%s/connection/%d/' % (API_VERSION, connection_id), headers=self.headers)
if resp_dict['all']:
return Connection.from_dict(resp_dict['all'][0])
return None
def create_connection(self, connection):
if not connection.connector:
connection.connector = self.get_connectors()[0].con_forms
if not connection.framework:
connection.framework = self.get_framework().con_forms
connection.creation_date = int( round(time.time() * 1000) )
connection.update_date = connection.creation_date
connection_dict = connection.to_dict()
request_dict = {
'all': [connection_dict]
}
resp = self._root.post('%s/connection/' % API_VERSION, data=json.dumps(request_dict), headers=self.headers)
if 'id' not in resp:
raise SqoopConnectionException.from_dict(resp)
connection.id = resp['id']
return connection
def update_connection(self, connection):
""" Update a connection """
if not connection.connector:
connection.connector = self.get_connectors()[0].con_forms
if not connection.framework:
connection.framework = self.get_framework().con_forms
connection.updated = int( round(time.time() * 1000) )
connection_dict = connection.to_dict()
request_dict = {
'all': [connection_dict]
}
resp = self._root.put('%s/connection/%d/' % (API_VERSION, connection.id), data=json.dumps(request_dict), headers=self.headers)
if resp['connector']['status'] in SqoopClient.STATUS_BAD or resp['framework']['status'] in SqoopClient.STATUS_BAD:
raise SqoopConnectionException.from_dict(resp)
return connection
def delete_connection(self, connection):
resp = self._root.delete('%s/connection/%d/' % (API_VERSION, connection.id), headers=self.headers)
return None
def get_jobs(self):
resp_dict = self._root.get('%s/job/all' % API_VERSION, headers=self.headers)
jobs = [Job.from_dict(job_dict) for job_dict in resp_dict['all']]
return jobs
def get_job(self, job_id):
resp_dict = self._root.get('%s/job/%d/' % (API_VERSION, job_id), headers=self.headers)
if resp_dict['all']:
return Job.from_dict(resp_dict['all'][0])
return None
def create_job(self, job):
if not job.connector:
job.connector = self.get_connectors()[0].job_forms[job.type.upper()]
if not job.framework:
job.framework = self.get_framework().job_forms[job.type.upper()]
job.creation_date = int( round(time.time() * 1000) )
job.update_date = job.creation_date
job_dict = job.to_dict()
request_dict = {
'all': [job_dict]
}
resp = self._root.post('%s/job/' % API_VERSION, data=json.dumps(request_dict), headers=self.headers)
if 'id' not in resp:
raise SqoopJobException.from_dict(resp)
job.id = resp['id']
return job
def update_job(self, job):
if not job.connector:
job.connector = self.get_connectors()[0].job_forms[job.type.upper()]
if not job.framework:
job.framework = self.get_framework().job_forms[job.type.upper()]
job.updated = int( round(time.time() * 1000) )
job_dict = job.to_dict()
request_dict = {
'all': [job_dict]
}
resp = self._root.put('%s/job/%d/' % (API_VERSION, job.id), data=json.dumps(request_dict), headers=self.headers)
if resp['connector']['status'] in SqoopClient.STATUS_BAD or resp['framework']['status'] in SqoopClient.STATUS_BAD:
raise SqoopJobException.from_dict(resp)
return job
def delete_job(self, job):
resp_dict = self._root.delete('%s/job/%d/' % (API_VERSION, job.id), headers=self.headers)
return None
def get_job_status(self, job):
resp_dict = self._root.get('%s/submission/action/%d/' % (API_VERSION, job.id), headers=self.headers)
return Submission.from_dict(resp_dict['all'][0])
def start_job(self, job):
resp_dict = self._root.post('%s/submission/action/%d/' % (API_VERSION, job.id), headers=self.headers)
if resp_dict['all'][0]['status'] in SqoopClient.STATUS_BAD:
raise SqoopSubmissionException.from_dict(resp_dict['all'][0])
return Submission.from_dict(resp_dict['all'][0])
def stop_job(self, job):
resp_dict = self._root.delete('%s/submission/action/%d/' % (API_VERSION, job.id), headers=self.headers)
return Submission.from_dict(resp_dict['all'][0])
def get_submissions(self):
resp_dict = self._root.get('%s/submission/history/all' % API_VERSION, headers=self.headers)
submissions = [Submission.from_dict(submission_dict) for submission_dict in resp_dict['all']]
return submissions
def set_user(self, user):
self._user = user
def set_language(self, language):
self._language = language
class SqoopClient(object):
STATUS_GOOD = ('FINE', 'ACCEPTABLE')
STATUS_BAD = ('UNACCEPTABLE', 'FAILURE_ON_SUBMIT')
def __init__(self, url, username, language='en'):
self._url = url
self._client = HttpClient(self._url, logger=LOG)
self._root = SqoopResource(self._client)
self._language = language
self._username = username
def __str__(self):
return "SqoopClient at %s" % self._url
@property
def url(self):
return self._url
@property
def headers(self):
return {
'Accept': 'application/json',
'Accept-Language': self._language,
'sqoop-user-name': self._username
}
def get_version(self):
return self._root.get('version', headers=self.headers)
def get_driver(self):
resp_dict = self._root.get('%s/driver' % API_VERSION, headers=self.headers)
driver = Driver.from_dict(resp_dict)
return driver
def get_connectors(self):
resp_dict = self._root.get('%s/connectors' % API_VERSION, headers=self.headers)
connectors = [ Connector.from_dict(connector_dict) for connector_dict in resp_dict['connectors'] ]
return connectors
def get_connector(self, connector_id):
resp_dict = self._root.get('%s/connector/%d/' % (API_VERSION, connector_id), headers=self.headers)
if resp_dict['connector']:
return Connector.from_dict(resp_dict['connector'])
return None
def get_links(self):
resp_dict = self._root.get('%s/links' % API_VERSION, headers=self.headers)
links = [Link.from_dict(link_dict) for link_dict in resp_dict['links']]
return links
def get_link(self, link_id):
resp_dict = self._root.get('%s/link/%d/' % (API_VERSION, link_id), headers=self.headers)
if resp_dict['link']:
return Link.from_dict(resp_dict['link'])
return None
def create_link(self, link):
link.creation_date = int( round(time.time() * 1000) )
link.update_date = link.creation_date
link_dict = link.to_dict()
request_dict = {
'link': link_dict
}
resp = self._root.post('%s/link/' % API_VERSION, data=json.dumps(request_dict), headers=self.headers)
# Lame check that iterates to make sure we have an error
# Server responds with: {'validation-result': [{},{}]} or {'validation-result': [{KEY: ERROR},{KEY: ERROR}]}
for result in resp['validation-result']:
if result:
raise SqoopException.from_dicts(resp['validation-result'])
link.id = resp['id']
return link
def update_link(self, link):
if not link.link_config_values:
link.link_config_values = self.get_connectors()[0].link_config
link.updated = int( round(time.time() * 1000) )
link_dict = link.to_dict()
request_dict = {
'link': link_dict
}
resp = self._root.put('%s/link/%d/' % (API_VERSION, link.id), data=json.dumps(request_dict), headers=self.headers)
# Lame check that iterates to make sure we have an error
# Server responds with: {'validation-result': [{},{}]} or {'validation-result': [{KEY: ERROR},{KEY: ERROR}]}
for result in resp['validation-result']:
if result:
raise SqoopException.from_dicts(resp['validation-result'])
return link
def delete_link(self, link):
resp = self._root.delete('%s/link/%d/' % (API_VERSION, link.id), headers=self.headers)
return None
def get_jobs(self):
resp_dict = self._root.get('%s/jobs' % API_VERSION, headers=self.headers)
jobs = [Job.from_dict(job_dict) for job_dict in resp_dict['jobs']]
return jobs
def get_job(self, job_id):
resp_dict = self._root.get('%s/job/%d/' % (API_VERSION, job_id), headers=self.headers)
if resp_dict['job']:
return Job.from_dict(resp_dict['job'])
return None
def create_job(self, job):
if not job.from_config_values:
job.from_config_values = self.get_connectors()[0].job_config['FROM']
if not job.to_config_values:
job.to_config_values = self.get_connectors()[0].job_config['TO']
if not job.driver_config_values:
job.driver_config_values = self.get_driver().job_config
job.creation_date = int( round(time.time() * 1000) )
job.update_date = job.creation_date
job_dict = job.to_dict()
request_dict = {
'job': job_dict
}
resp = self._root.post('%s/job/' % API_VERSION, data=json.dumps(request_dict), headers=self.headers)
if 'id' not in resp:
raise SqoopException.from_dicts(resp['validation-result'])
job.id = resp['id']
return job
def update_job(self, job):
if not job.from_config_values:
job.from_config_values = self.get_connectors()[0].job_config['FROM']
if not job.to_config_values:
job.to_config_values = self.get_connectors()[0].job_config['TO']
if not job.driver_config_values:
job.driver_config_values = self.get_driver().job_config
job.updated = int( round(time.time() * 1000) )
job_dict = job.to_dict()
request_dict = {
'job': job_dict
}
resp = self._root.put('%s/job/%d/' % (API_VERSION, job.id), data=json.dumps(request_dict), headers=self.headers)
# Lame check that iterates to make sure we have an error
# Server responds with: {'validation-result': [{},{}]} or {'validation-result': [{KEY: ERROR},{KEY: ERROR}]}
for result in resp['validation-result']:
if result:
raise SqoopException.from_dicts(resp['validation-result'])
return job
def delete_job(self, job):
resp_dict = self._root.delete('%s/job/%s' % (API_VERSION, job.id), headers=self.headers)
return None
def get_job_status(self, job):
resp_dict = self._root.get('%s/job/%d/status' % (API_VERSION, job.id), headers=self.headers)
return Submission.from_dict(resp_dict['submission'])
def start_job(self, job):
resp_dict = self._root.put('%s/job/%d/start' % (API_VERSION, job.id), headers=self.headers)
if resp_dict['submission']['status'] in SqoopClient.STATUS_BAD:
raise SqoopSubmissionException.from_dict(resp_dict['submission'])
return Submission.from_dict(resp_dict['submission'])
def stop_job(self, job):
resp_dict = self._root.put('%s/job/%d/stop' % (API_VERSION, job.id), headers=self.headers)
return Submission.from_dict(resp_dict['submission'])
def get_submissions(self):
resp_dict = self._root.get('%s/submissions' % API_VERSION, headers=self.headers)
submissions = [Submission.from_dict(submission_dict) for submission_dict in resp_dict['submissions']]
return submissions
def set_user(self, user):
self._user = user
def set_language(self, language):
self._language = language
class SqoopClient(object):
"""
Sqoop client
"""
STATUS_GOOD = ('FINE', 'ACCEPTABLE')
STATUS_BAD = ('UNACCEPTABLE', 'FAILURE_ON_SUBMIT')
def __init__(self, url, username, language='en'):
self._url = url
self._client = HttpClient(self._url, logger=LOG)
self._root = SqoopResource(self._client)
self._language = language
self._username = username
def __str__(self):
return "SqoopClient at %s" % self._url
@property
def url(self):
return self._url
@property
def headers(self):
return {
'Accept': 'application/json',
'Accept-Language': self._language,
'sqoop-user-name': self._username
}
def get_version(self):
return self._root.get('version', headers=self.headers)
def get_framework(self):
resp_dict = self._root.get('%s/framework/all' % API_VERSION, headers=self.headers)
framework = Framework.from_dict(resp_dict)
return framework
def get_connectors(self):
resp_dict = self._root.get('%s/connector/all' % API_VERSION, headers=self.headers)
connectors = [ Connector.from_dict(connector_dict, resp_dict['resources-connector']) for connector_dict in resp_dict['all'] ]
return connectors
def get_connector(self, connector_id):
resp_dict = self._root.get('%s/connector/%d/' % (API_VERSION, connector_id), headers=self.headers)
if resp_dict['all']:
return Connector.from_dict(resp_dict['all'][0], resp_dict['resources-connector'])
return None
def get_connections(self):
resp_dict = self._root.get('%s/connection/all' % API_VERSION, headers=self.headers)
connections = [Connection.from_dict(conn_dict) for conn_dict in resp_dict['all']]
return connections
def get_connection(self, connection_id):
resp_dict = self._root.get('%s/connection/%d/' % (API_VERSION, connection_id), headers=self.headers)
if resp_dict['all']:
return Connection.from_dict(resp_dict['all'][0])
return None
def create_connection(self, connection):
if not connection.connector:
connection.connector = self.get_connectors()[0].con_forms
if not connection.framework:
connection.framework = self.get_framework().con_forms
connection.created = int( round(time.time() * 1000) )
connection.updated = connection.created
connection_dict = connection.to_dict()
request_dict = {
'all': [connection_dict]
}
resp = self._root.post('%s/connection/' % API_VERSION, data=json.dumps(request_dict), headers=self.headers)
if 'id' not in resp:
raise SqoopConnectionException.from_dict(resp)
connection.id = resp['id']
return connection
def update_connection(self, connection):
""" Update a connection """
if not connection.connector:
connection.connector = self.get_connectors()[0].con_forms
if not connection.framework:
connection.framework = self.get_framework().con_forms
connection.updated = int( round(time.time() * 1000) )
connection_dict = connection.to_dict()
request_dict = {
'all': [connection_dict]
}
resp = self._root.put('%s/connection/%d/' % (API_VERSION, connection.id), data=json.dumps(request_dict), headers=self.headers)
if resp['connector']['status'] in SqoopClient.STATUS_BAD or resp['framework']['status'] in SqoopClient.STATUS_BAD:
raise SqoopConnectionException.from_dict(resp)
return connection
def delete_connection(self, connection):
resp = self._root.delete('%s/connection/%d/' % (API_VERSION, connection.id), headers=self.headers)
return None
def get_jobs(self):
resp_dict = self._root.get('%s/job/all' % API_VERSION, headers=self.headers)
jobs = [Job.from_dict(job_dict) for job_dict in resp_dict['all']]
return jobs
def get_job(self, job_id):
resp_dict = self._root.get('%s/job/%d/' % (API_VERSION, job_id), headers=self.headers)
if resp_dict['all']:
return Job.from_dict(resp_dict['all'][0])
return None
def create_job(self, job):
if not job.connector:
job.connector = self.get_connectors()[0].job_forms[job.type.upper()]
if not job.framework:
job.framework = self.get_framework().job_forms[job.type.upper()]
job.created = int( round(time.time() * 1000) )
job.updated = job.created
job_dict = job.to_dict()
request_dict = {
'all': [job_dict]
}
resp = self._root.post('%s/job/' % API_VERSION, data=json.dumps(request_dict), headers=self.headers)
if 'id' not in resp:
raise SqoopJobException.from_dict(resp)
job.id = resp['id']
return job
def update_job(self, job):
if not job.connector:
job.connector = self.get_connectors()[0].job_forms[job.type.upper()]
if not job.framework:
job.framework = self.get_framework().job_forms[job.type.upper()]
job.updated = int( round(time.time() * 1000) )
job_dict = job.to_dict()
request_dict = {
'all': [job_dict]
}
resp = self._root.put('%s/job/%d/' % (API_VERSION, job.id), data=json.dumps(request_dict), headers=self.headers)
if resp['connector']['status'] in SqoopClient.STATUS_BAD or resp['framework']['status'] in SqoopClient.STATUS_BAD:
raise SqoopJobException.from_dict(resp)
return job
def delete_job(self, job):
resp_dict = self._root.delete('%s/job/%d/' % (API_VERSION, job.id), headers=self.headers)
return None
def get_job_status(self, job):
resp_dict = self._root.get('%s/submission/action/%d/' % (API_VERSION, job.id), headers=self.headers)
return Submission.from_dict(resp_dict['all'][0])
def start_job(self, job):
resp_dict = self._root.post('%s/submission/action/%d/' % (API_VERSION, job.id), headers=self.headers)
if resp_dict['all'][0]['status'] in SqoopClient.STATUS_BAD:
raise SqoopSubmissionException.from_dict(resp_dict['all'][0])
return Submission.from_dict(resp_dict['all'][0])
def stop_job(self, job):
resp_dict = self._root.delete('%s/submission/action/%d/' % (API_VERSION, job.id), headers=self.headers)
return Submission.from_dict(resp_dict['all'][0])
def get_submissions(self):
resp_dict = self._root.get('%s/submission/history/all' % API_VERSION, headers=self.headers)
submissions = [Submission.from_dict(submission_dict) for submission_dict in resp_dict['all']]
return submissions
def set_user(self, user):
self._user = user
def set_language(self, language):
self._language = language