def test_query_all_include_deleted(self):
"""
Test that we query all and fetch additional result sets automatically.
"""
responses.add(
responses.GET,
re.compile(r'^https://.*/queryAll/\?q=SELECT\+ID\+FROM\+Account$'),
body='{"records": [{"ID": "1"}], "done": false, "nextRecordsUrl": '
'"https://example.com/queryAll/next-records-id"}',
status=http.OK)
responses.add(
responses.GET,
re.compile(r'^https://.*/queryAll/next-records-id$'),
body='{"records": [{"ID": "2"}], "done": true}',
status=http.OK)
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session)
result = client.query_all('SELECT ID FROM Account',
include_deleted=True)
self.assertEqual(
result,
OrderedDict([(u'records', [
OrderedDict([(u'ID', u'1')]),
OrderedDict([(u'ID', u'2')])
]), (u'done', True)]))
def test_query_include_deleted(self):
"""Test querying for all records generates the expected request"""
responses.add(
responses.GET,
re.compile(r'^https://.*/queryAll/\?q=SELECT\+ID\+FROM\+Account$'),
body='{}',
status=http.OK)
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session)
result = client.query('SELECT ID FROM Account', include_deleted=True)
self.assertEqual(result, {})
def test_query_more_id_not_url(self):
"""
Test fetching additional results by ID generates the expected request
"""
responses.add(
responses.GET,
re.compile(r'^https://.*/query/next-records-id$'),
body='{}',
status=http.OK)
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session)
result = client.query_more('next-records-id', identifier_is_url=False)
self.assertEqual(result, {})
def test_api_limits(self):
"""Test method for getting Salesforce organization limits"""
responses.add(
responses.GET,
re.compile(r'^https://.*/limits/$'),
json=tests.ORGANIZATION_LIMITS_RESPONSE,
status=http.OK
)
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session)
result = client.limits()
self.assertEqual(result, tests.ORGANIZATION_LIMITS_RESPONSE)
def test_proxies_inherited_set_on_session(self):
"""Test Salesforce and SFType use same custom proxies"""
session = requests.Session()
session.proxies = tests.PROXIES
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session)
self.assertIs(tests.PROXIES, client.session.proxies)
self.assertIs(tests.PROXIES, client.Contact.session.proxies)
def test_proxies_inherited_default(self):
"""Test Salesforce and SFType use same proxies"""
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session)
self.assertIs(session.proxies, client.session.proxies)
self.assertIs(session.proxies, client.Contact.session.proxies)
def test_shared_custom_session_to_sftype(self):
"""Test Salesforce and SFType instances share custom `Session`"""
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session)
self.assertIs(session, client.session)
self.assertIs(session, client.Contact.session)
def test_md_deploy_failed_status_code(self, mock_read_zip):
""""
Test method for metadata deployment
"""
#pylint: disable=unused-argument
responses.add(
responses.POST,
re.compile(r'^https://.*'),
body="Unrecognized Error",
status=2599
)
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance=tests.INSTANCE_URL,
session=session)
with self.assertRaises(Exception):
client.deploy("path/to/fake/zip.zip", sandbox=False)
def test_api_limits(self):
"""Test method for getting Salesforce organization limits"""
responses.add(
responses.GET,
re.compile(r'^https://.*/limits/$'),
json=tests.ORGANIZATION_LIMITS_RESPONSE,
status=http.OK
)
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session)
result = client.limits()
self.assertEqual(result, tests.ORGANIZATION_LIMITS_RESPONSE)
def test_query_parse_float(self):
"""Test parsing of floating point numbers as floats"""
responses.add(
responses.GET,
re.compile(r'^https://.*/query/\?q=SELECT\+ID%2C\+Price\+'
r'FROM\+Account$'),
body='{"records": [{"ID": "1", "Price": 13.40},'
'{"ID": "2", "Price": 1.12345678901234567},'
'{"ID": "3", "Price": 123456789012345678},'
'{"ID": "4", "Price": 0},'
'{"ID": "5", "Price": -1.1234567890123456},'
'{"ID": "6", "Price": -12345678901234567}],'
'"done": false, "nextRecordsUrl": '
'"https://example.com/query/next-records-id", "totalSize": 6}',
status=http.OK)
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session)
result = client.query('SELECT ID, Price FROM Account')
self.assertEqual(
result,
OrderedDict([
('records', [
OrderedDict([('ID', '1'), ('Price', 13.4)]),
OrderedDict([
('ID', '2'), ('Price', 1.12345678901234567)
]),
OrderedDict([
('ID', '3'), ('Price', 123456789012345678)
]),
OrderedDict([('ID', '4'), ('Price', 0)]),
OrderedDict([
('ID', '5'), ('Price', -1.1234567890123456),
]),
OrderedDict([
('ID', '6'), ('Price', -12345678901234567),
]),
]),
('done', False),
('nextRecordsUrl', "https://example.com/query/next-records-id"),
('totalSize', 6),
]))
def test_delete(self):
"""Test delete records"""
operation = 'delete'
responses.add(
responses.POST,
re.compile(r'^https://[^/job].*/job$'),
body='{"apiVersion": 42.0, "concurrencyMode": "Parallel",'
'"contentType": "JSON","id": "Job-1","object": "Contact",'
'"operation": "%s","state": "Open"}' % operation,
status=http.OK)
responses.add(
responses.POST,
re.compile(r'^https://[^/job].*/job/Job-1/batch$'),
body='{"id": "Batch-1","jobId": "Job-1","state": "Queued"}',
status=http.OK
)
responses.add(
responses.POST,
re.compile(r'^https://[^/job].*/job/Job-1$'),
body='{"apiVersion" : 42.0, "concurrencyMode" : "Parallel",'
'"contentType" : "JSON","id" : "Job-1","object" : "Contact",'
'"operation" : "%s","state" : "Closed"}' % operation,
status=http.OK
)
responses.add(
responses.GET,
re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1$'),
body='{"id": "Batch-1","jobId": "Job-1","state": "InProgress"}',
status=http.OK
)
responses.add(
responses.GET,
re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1$'),
body='{"id": "Batch-1","jobId": "Job-1","state": "Completed"}',
status=http.OK
)
responses.add(
responses.GET,
re.compile(
r'^https://[^/job].*/job/Job-1/batch/Batch-1/result$'),
body='[{"success": true,"created": true,"id": "001xx000003DHP0AAO",'
'"errors": []},{"success": true,"created": true,'
'"id": "001xx000003DHP1AAO","errors": []}]',
status=http.OK
)
data = [{
'id': 'ID-1',
}, {
'id': 'ID-2',
}]
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session)
contact = client.bulk.Contact.delete(data)
self.assertEqual(self.expected, contact)
def test_proxies_ignored(self):
"""Test overridden proxies are ignored"""
session = requests.Session()
session.proxies = tests.PROXIES
with patch('simple_salesforce.api.logger.warning') as mock_log:
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL, session=session, proxies={})
self.assertIn('ignoring proxies', mock_log.call_args[0][0])
self.assertIs(tests.PROXIES, client.session.proxies)
def test_bulk_type(self):
"""Test bulk type creation"""
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session)
contact = client.bulk.Contact
self.assertIs('Contact', contact.object_name)
self.assertIs(client.bulk_url, contact.bulk_url)
self.assertEqual(tests.BULK_HEADERS, contact.headers)
def test_bulk_handler(self):
"""Test bulk handler creation"""
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session)
bulk_handler = client.bulk
self.assertIs(tests.SESSION_ID, bulk_handler.session_id)
self.assertIs(client.bulk_url, bulk_handler.bulk_url)
self.assertEqual(tests.BULK_HEADERS, bulk_handler.headers)
def test_check_status_payload_error(self, mock_post):
""" "
Test method for metadata deployment
"""
mock_response = Mock()
mock_response.status_code = 200
mock_response.text = '<?xml version="1.0" encoding="UTF-8"?><soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns="http://soap.sforce.com/2006/04/metadata"><soapenv:Body><checkDeployStatusResponse><result><checkOnly>true</checkOnly><completedDate>2020-10-28T13:37:48.000Z</completedDate><createdBy>0053D0000052Xaq</createdBy><createdByName>User User</createdByName><createdDate>2020-10-28T13:37:46.000Z</createdDate><details><componentFailures><changed>false</changed><componentType></componentType><created>false</created><createdDate>2020-10-28T13:37:47.000Z</createdDate><deleted>false</deleted><fileName>package.xml</fileName><fullName>package.xml</fullName><problem>No package.xml found</problem><problemType>Error</problemType><success>false</success></componentFailures><runTestResult><numFailures>0</numFailures><numTestsRun>0</numTestsRun><totalTime>0.0</totalTime></runTestResult></details><done>true</done><id>0Af3D00001NVD0TSAX</id><ignoreWarnings>false</ignoreWarnings><lastModifiedDate>2020-10-28T13:37:48.000Z</lastModifiedDate><numberComponentErrors>0</numberComponentErrors><numberComponentsDeployed>0</numberComponentsDeployed><numberComponentsTotal>0</numberComponentsTotal><numberTestErrors>0</numberTestErrors><numberTestsCompleted>0</numberTestsCompleted><numberTestsTotal>0</numberTestsTotal><rollbackOnError>true</rollbackOnError><runTestsEnabled>false</runTestsEnabled><startDate>2020-10-28T13:37:47.000Z</startDate><status>Failed</status><success>false</success></result></checkDeployStatusResponse></soapenv:Body></soapenv:Envelope>'
mock_post.return_value = mock_response
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance=tests.SERVER_URL,
session=session)
state, state_detail, deployment_detail, unit_test_detail = client.checkDeployStatus(
"abdcefg")
self.assertEqual(state, "Failed")
self.assertEqual(state_detail, None)
self.assertEqual(
deployment_detail,
{
"total_count":
"0",
"failed_count":
"0",
"deployed_count":
"0",
"errors": [{
"type": None,
"file": "package.xml",
"status": "Error",
"message": "No package.xml found"
}],
},
)
self.assertEqual(
unit_test_detail, {
"total_count": "0",
"failed_count": "0",
"completed_count": "0",
"errors": []
})
def test_query_more_parse_float_to_decimal(self):
"""Test querying generates float as Decimal values"""
responses.add(
responses.GET,
re.compile(r'^https://.*/query/next-records-id$'),
body='{"currency": 1.0}',
status=http.OK,
)
session = requests.Session()
client = Salesforce(
session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session,
parse_float=decimal.Decimal,
)
result = client.query_more('next-records-id', identifier_is_url=False)
self.assertIsInstance(result["currency"], decimal.Decimal)
self.assertNotIsInstance(result["currency"], float)
self.assertEqual(result, {"currency": decimal.Decimal(1.0)})
self.assertEqual(result, {"currency": 1.0})
self.assertNotEqual(result, {"currency": "1.0"})
class SalesforceExtractor(Extractor):
_incoming_table_class = IncomingSalesforceRecord
def __init__(self, username, password, token):
'''
Constructor
'''
self.api = Salesforce(username=username, password=password, security_token=token)
super(SalesforceExtractor, self).__init__(self._incoming_table_class)
def do_extract(self):
res = self.api.describe()
obj_names = [obj['name'] for obj in res['sobjects']]
obj_fields = dict()
for obj in obj_names:
api_method_call = getattr(self.api, obj)
try:
obj_meta = api_method_call.describe()
# get field names for everything except the content blobs
# if there's a blob in the record queries will return only one at a time i.e. one API call per record
field_names = [field['name'] for field in obj_meta['fields'] if field['type'] not in ('base64')]
obj_fields[obj] = field_names
except SalesforceMalformedRequest, m:
logger.warn('Got SalesforceMalformedRequest when querying metadata for object %s' % obj)
logger.warn(str(m))
for obj in obj_fields.keys():
fieldstr = ','.join(obj_fields[obj])
querystr = 'SELECT %s FROM %s' % (fieldstr,obj)
logger.info('Executing Salesforce query %s' % querystr)
records = []
try:
obj_res = self.api.query(querystr)
if 'records' in obj_res:
records.extend(obj_res['records'])
while 'done' in obj_res and not obj_res['done']:
obj_res = self.api.query_more(obj_res['nextRecordsUrl'], True)
if 'records' in obj_res:
records.extend(obj_res['records'])
logger.info('%d results of type %s' % (len(records),obj))
for rec in records:
IncomingSalesforceRecord.create(sf_id=id,object_type=obj,record=json.dumps(rec))
except SalesforceMalformedRequest, m:
logger.warn('Got SalesforceMalformedRequest when trying to retrieve all fields for object %s' % obj)
logger.warn(str(m))
def test_query_parse_float_to_decimal(self):
"""Test querying generates float as Decimal values"""
responses.add(
responses.GET,
re.compile(
r'^https://.*/query/\?q=SELECT\+currency\+FROM\+Account$'
),
body='{"currency": 1.0}',
status=http.OK,
)
session = requests.Session()
client = Salesforce(
session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session,
parse_float=decimal.Decimal,
)
result = client.query('SELECT currency FROM Account')
self.assertIsInstance(result["currency"], decimal.Decimal)
self.assertNotIsInstance(result["currency"], float)
self.assertEqual(result, {"currency": decimal.Decimal(1.0)})
self.assertEqual(result, {"currency": 1.0})
self.assertNotEqual(result, {"currency": "1.0"})
def test_api_usage_simple(self):
"""Make sure a header response is recorded"""
responses.add(
responses.GET,
re.compile(r'^https://.*$'),
body='{"example": 1}',
adding_headers={"Sforce-Limit-Info": "api-usage=18/5000"},
status=http.OK)
client = Salesforce.__new__(Salesforce)
client.request = requests.Session()
client.headers = {}
client.base_url = 'https://localhost'
client.query('q')
self.assertDictEqual(client.api_usage, {'api-usage': Usage(18, 5000)})
def test_api_usage_simple(self):
"""Make sure a header response is recorded"""
responses.add(
responses.GET,
re.compile(r'^https://.*$'),
body='{"example": 1}',
adding_headers={"Sforce-Limit-Info": "api-usage=18/5000"},
status=http.OK
)
client = Salesforce.__new__(Salesforce)
client.request = requests.Session()
client.headers = {}
client.base_url = 'https://localhost'
client.query('q')
self.assertDictEqual(client.api_usage, {'api-usage': Usage(18, 5000)})
def test_custom_version_success(self):
"""Test custom version"""
responses.add(responses.POST,
re.compile(r'^https://.*$'),
body=tests.LOGIN_RESPONSE_SUCCESS,
status=http.OK)
# Use an invalid version that is guaranteed to never be used
expected_version = '4.2'
client = Salesforce(session=requests.Session(),
username='******',
password='******',
security_token='token',
version=expected_version)
self.assertEqual(
client.base_url.split('/')[-2], 'v%s' % expected_version)
def test_custom_version_success(self):
"""Test custom version"""
responses.add(responses.POST,
re.compile(r"^https://.*$"),
body=tests.LOGIN_RESPONSE_SUCCESS,
status=http.OK)
# Use an invalid version that is guaranteed to never be used
expected_version = "4.2"
client = Salesforce(
session=requests.Session(),
username="******",
password="******",
security_token="token",
version=expected_version,
)
self.assertEqual(
client.base_url.split("/")[-2], "v%s" % expected_version)
def test_query_fail(self):
"""Test bulk query records failure"""
operation = 'query'
responses.add(
responses.POST,
re.compile(r'^https://[^/job].*/job$'),
body='{"apiVersion": 42.0, "concurrencyMode": "Parallel",'
'"contentType": "JSON","id": "Job-1","object": "Contact",'
'"operation": "%s","state": "Open"}' % operation,
status=http.OK)
responses.add(
responses.POST,
re.compile(r'^https://[^/job].*/job/Job-1/batch$'),
body='{"id": "Batch-1","jobId": "Job-1","state": "Queued"}',
status=http.OK)
responses.add(
responses.POST,
re.compile(r'^https://[^/job].*/job/Job-1$'),
body='{"apiVersion" : 42.0, "concurrencyMode" : "Parallel",'
'"contentType" : "JSON","id" : "Job-1","object" : "Contact",'
'"operation" : "%s","state" : "Closed"}' % operation,
status=http.OK)
responses.add(
responses.GET,
re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1$'),
body='{"id": "Batch-1","jobId": "Job-1","state": "InProgress"}',
status=http.OK)
responses.add(
responses.GET,
re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1$'),
body='{"id": "Batch-1","jobId": "Job-1","state": "Failed",'
'"stateMessage": "InvalidBatch : Failed to process query"}',
status=http.OK)
data = 'SELECT ASDFASfgsds FROM Contact'
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session)
self.assertRaises(SalesforceGeneralError, client.bulk.Contact.query,
data)
def test_api_usage_per_app(self):
"""Make sure a header response is recorded"""
pau = "api-usage=25/5000; per-app-api-usage=17/250(appName=sample-app)"
responses.add(responses.GET,
re.compile(r'^https://.*$'),
body='{"example": 1}',
adding_headers={"Sforce-Limit-Info": pau},
status=http.OK)
client = Salesforce.__new__(Salesforce)
client.request = requests.Session()
client.headers = {}
client.base_url = 'https://localhost'
client.query('q')
self.assertDictEqual(
client.api_usage, {
'api-usage': Usage(25, 5000),
'per-app-api-usage': PerAppUsage(17, 250, 'sample-app')
})
def test_api_usage_per_app(self):
"""Make sure a header response is recorded"""
pau = "api-usage=25/5000; per-app-api-usage=17/250(appName=sample-app)"
responses.add(
responses.GET,
re.compile(r'^https://.*$'),
body='{"example": 1}',
adding_headers={"Sforce-Limit-Info": pau},
status=http.OK
)
client = Salesforce.__new__(Salesforce)
client.request = requests.Session()
client.headers = {}
client.base_url = 'https://localhost'
client.query('q')
self.assertDictEqual(client.api_usage,
{'api-usage': Usage(25, 5000),
'per-app-api-usage': PerAppUsage(17, 250,
'sample-app')})
def test_custom_session_success(self):
httpretty.register_uri(httpretty.POST,
re.compile(r'^https://.*$'),
body=tests.LOGIN_RESPONSE_SUCCESS,
status=http.OK)
session_state = {
'called': False,
}
def on_response(*args, **kwargs):
session_state['called'] = True
session = requests.Session()
session.hooks = {
'response': on_response,
}
sf = Salesforce(session=session,
username='******',
password='******',
security_token='token')
self.assertEqual(tests.SESSION_ID, sf.session_id)
self.assertEqual(session, sf.request)
def test_check_status_payload_error(self):
""""
Test method for metadata deployment
"""
# pylint: disable-msg=C0301
mock_response = '<?xml version="1.0" ' \
'encoding="UTF-8"?><soapenv:Envelope ' \
'xmlns:soapenv="http://schemas.xmlsoap.org/soap' \
'/envelope/" ' \
'xmlns="http://soap.sforce.com/2006/04/metadata' \
'"><soapenv:Body><checkDeployStatusResponse><result' \
'><checkOnly>true</checkOnly><completedDate>2020-10' \
'-28T13:37:48.000Z</completedDate><createdBy' \
'>0053D0000052Xaq</createdBy><createdByName>User ' \
'User</createdByName><createdDate>2020-10-28T13:37:46' \
'.000Z</createdDate><details><componentFailures' \
'><changed>false</changed><componentType' \
'></componentType><created>false</created' \
'><createdDate>2020-10-28T13:37:47.000Z</createdDate' \
'><deleted>false</deleted><fileName>package.xml' \
'</fileName><fullName>package.xml</fullName><problem' \
'>No package.xml ' \
'found</problem><problemType>Error</problemType' \
'><success>false</success></componentFailures' \
'><runTestResult><numFailures>0</numFailures' \
'><numTestsRun>0</numTestsRun><totalTime>0.0' \
'</totalTime></runTestResult></details><done>true' \
'</done><id>0Af3D00001NVD0TSAX</id><ignoreWarnings' \
'>false</ignoreWarnings><lastModifiedDate>2020-10' \
'-28T13:37:48.000Z</lastModifiedDate' \
'><numberComponentErrors>0</numberComponentErrors' \
'><numberComponentsDeployed>0</numberComponentsDeployed><numberComponentsTotal>0</numberComponentsTotal><numberTestErrors>0</numberTestErrors><numberTestsCompleted>0</numberTestsCompleted><numberTestsTotal>0</numberTestsTotal><rollbackOnError>true</rollbackOnError><runTestsEnabled>false</runTestsEnabled><startDate>2020-10-28T13:37:47.000Z</startDate><status>Failed</status><success>false</success></result></checkDeployStatusResponse></soapenv:Body></soapenv:Envelope>'
responses.add(responses.POST,
re.compile(r'^https://.*/deployRequest/abdcefg'),
body=mock_response,
status=http.OK)
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance=tests.INSTANCE_URL,
session=session)
result = client.checkDeployStatus("abdcefg", sandbox=False)
self.assertEqual(result.get('state'), "Failed")
self.assertEqual(result.get('state_detail'), None)
self.assertEqual(
result.get('deployment_detail'), {
'total_count':
'0',
'failed_count':
'0',
'deployed_count':
'0',
'errors': [{
'type': None,
'file': 'package.xml',
'status': 'Error',
'message': 'No package.xml found'
}]
})
self.assertEqual(
result.get('unit_test_detail'), {
'total_count': '0',
'failed_count': '0',
'completed_count': '0',
'errors': []
})
def test_upsert(self):
"""Test upsert records"""
operation = 'upsert'
responses.add(
responses.POST,
re.compile(r'^https://[^/job].*/job$'),
body='{"apiVersion": 42.0, "concurrencyMode": "Parallel",'
'"contentType": "JSON","id": "Job-1","object": "Contact",'
'"operation": "%s","state": "Open"}' % operation,
status=http.OK)
responses.add(
responses.POST,
re.compile(r'^https://[^/job].*/job/Job-1/batch$'),
body='{"id": "Batch-1","jobId": "Job-1","state": "Queued"}',
status=http.OK
)
responses.add(
responses.POST,
re.compile(r'^https://[^/job].*/job/Job-1$'),
body='{"apiVersion" : 42.0, "concurrencyMode" : "Parallel",'
'"contentType" : "JSON","id" : "Job-1","object" : "Contact",'
'"operation" : "%s","state" : "Closed"}' % operation,
status=http.OK
)
responses.add(
responses.GET,
re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1$'),
body='{"id": "Batch-1","jobId": "Job-1","state": "InProgress"}',
status=http.OK
)
responses.add(
responses.GET,
re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1$'),
body='{"id": "Batch-1","jobId": "Job-1","state": "Completed"}',
status=http.OK
)
responses.add(
responses.GET,
re.compile(
r'^https://[^/job].*/job/Job-1/batch/Batch-1/result$'),
body='[{"success": true,"created": true,"id": "001xx000003DHP0AAO",'
'"errors": []},{"success": true,"created": true,'
'"id": "001xx000003DHP1AAO","errors": []}]',
status=http.OK
)
data = [{
'Custom_Id__c': 'CustomID1',
'AccountId': 'ID-13',
'Email': '*****@*****.**',
'FirstName': 'Bob',
'LastName': 'x'
}, {
'Custom_Id__c': 'CustomID2',
'AccountId': 'ID-24',
'Email': '*****@*****.**',
'FirstName': 'Alice',
'LastName': 'y'
}]
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session)
contact = client.bulk.Contact.upsert(data, 'Custom_Id__c')
self.assertEqual(self.expected, contact)
def test_shared_session_to_sftype(self):
"""Test Salesforce and SFType instances share default `Session`"""
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL)
self.assertIs(client.session, client.Contact.session)
def guess_field_types(request):
file_format = json.loads(request.POST.get('fileFormat', '{}'))
if file_format['inputFormat'] == 'file':
indexer = MorphlineIndexer(request.user, request.fs)
path = urllib_unquote(file_format["path"])
stream = request.fs.open(path)
encoding = chardet.detect(stream.read(10000)).get('encoding')
stream.seek(0)
_convert_format(file_format["format"], inverse=True)
format_ = indexer.guess_field_types({
"file": {
"stream": stream,
"name": path
},
"format": file_format['format']
})
# Note: Would also need to set charset to table (only supported in Hive)
if 'sample' in format_ and format_['sample']:
format_['sample'] = escape_rows(format_['sample'],
nulls_only=True,
encoding=encoding)
for col in format_['columns']:
col['name'] = smart_unicode(col['name'],
errors='replace',
encoding=encoding)
elif file_format['inputFormat'] == 'table':
sample = get_api(request, {
'type': 'hive'
}).get_sample_data({'type': 'hive'},
database=file_format['databaseName'],
table=file_format['tableName'])
db = dbms.get(request.user)
table_metadata = db.get_table(database=file_format['databaseName'],
table_name=file_format['tableName'])
format_ = {
"sample":
sample['rows'][:4],
"columns": [
Field(col.name,
HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type,
'string')).to_dict()
for col in table_metadata.cols
]
}
elif file_format['inputFormat'] == 'query':
query_id = file_format['query']['id'] if file_format['query'].get(
'id') else file_format['query']
notebook = Notebook(document=Document2.objects.document(
user=request.user, doc_id=query_id)).get_data()
snippet = notebook['snippets'][0]
db = get_api(request, snippet)
if file_format.get('sampleCols'):
columns = file_format.get('sampleCols')
sample = file_format.get('sample')
else:
snippet['query'] = snippet['statement']
try:
sample = db.fetch_result(notebook, snippet, 4,
start_over=True)['rows'][:4]
except Exception as e:
LOG.warn(
'Skipping sample data as query handle might be expired: %s'
% e)
sample = [[], [], [], [], []]
columns = db.autocomplete(snippet=snippet, database='', table='')
columns = [
Field(
col['name'],
HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'],
'string')).to_dict()
for col in columns['extended_columns']
]
format_ = {
"sample": sample,
"columns": columns,
}
elif file_format['inputFormat'] == 'rdbms':
api = _get_api(request)
sample = api.get_sample_data(None,
database=file_format['rdbmsDatabaseName'],
table=file_format['tableName'])
format_ = {
"sample":
list(sample['rows'])[:4],
"columns": [
Field(col['name'], col['type']).to_dict()
for col in sample['full_headers']
]
}
elif file_format['inputFormat'] == 'stream':
if file_format['streamSelection'] == 'kafka':
if file_format.get(
'kafkaSelectedTopics') == 'NavigatorAuditEvents':
kafkaFieldNames = [
'id', 'additionalInfo', 'allowed', 'collectionName',
'databaseName', 'db', 'DELEGATION_TOKEN_ID', 'dst',
'entityId', 'family', 'impersonator', 'ip', 'name',
'objectType', 'objType', 'objUsageType', 'operationParams',
'operationText', 'op', 'opText', 'path', 'perms',
'privilege', 'qualifier', 'QUERY_ID', 'resourcePath',
'service', 'SESSION_ID', 'solrVersion', 'src', 'status',
'subOperation', 'tableName', 'table', 'time', 'type',
'url', 'user'
]
kafkaFieldTypes = ['string'] * len(kafkaFieldNames)
kafkaFieldNames.append('timeDate')
kafkaFieldTypes.append('date')
else:
# Note: mocked here, should come from SFDC or Kafka API or sampling job
kafkaFieldNames = file_format.get('kafkaFieldNames',
'').split(',')
kafkaFieldTypes = file_format.get('kafkaFieldTypes',
'').split(',')
data = """%(kafkaFieldNames)s
%(data)s""" % {
'kafkaFieldNames': ','.join(kafkaFieldNames),
'data': '\n'.join(
[','.join(['...'] * len(kafkaFieldTypes))] * 5)
}
stream = string_io()
stream.write(data)
_convert_format(file_format["format"], inverse=True)
indexer = MorphlineIndexer(request.user, request.fs)
format_ = indexer.guess_field_types({
"file": {
"stream": stream,
"name": file_format['path']
},
"format": file_format['format']
})
type_mapping = dict(list(zip(kafkaFieldNames, kafkaFieldTypes)))
for col in format_['columns']:
col['keyType'] = type_mapping[col['name']]
col['type'] = type_mapping[col['name']]
elif file_format['streamSelection'] == 'flume':
if 'hue-httpd/access_log' in file_format['channelSourcePath']:
columns = [{
'name': 'id',
'type': 'string',
'unique': True
}, {
'name': 'client_ip',
'type': 'string'
}, {
'name': 'time',
'type': 'date'
}, {
'name': 'request',
'type': 'string'
}, {
'name': 'code',
'type': 'plong'
}, {
'name': 'bytes',
'type': 'plong'
}, {
'name': 'method',
'type': 'string'
}, {
'name': 'url',
'type': 'string'
}, {
'name': 'protocol',
'type': 'string'
}, {
'name': 'app',
'type': 'string'
}, {
'name': 'subapp',
'type': 'string'
}]
else:
columns = [{'name': 'message', 'type': 'string'}]
format_ = {
"sample": [['...'] * len(columns)] * 4,
"columns": [
Field(col['name'],
HiveFormat.FIELD_TYPE_TRANSLATE.get(
col['type'], 'string'),
unique=col.get('unique')).to_dict()
for col in columns
]
}
elif file_format['inputFormat'] == 'connector':
if file_format['connectorSelection'] == 'sfdc':
sf = Salesforce(username=file_format['streamUsername'],
password=file_format['streamPassword'],
security_token=file_format['streamToken'])
table_metadata = [{
'name': column['name'],
'type': column['type']
} for column in sf.restful('sobjects/%(streamObject)s/describe/' %
file_format)['fields']]
query = 'SELECT %s FROM %s LIMIT 4' % (', '.join(
[col['name']
for col in table_metadata]), file_format['streamObject'])
print(query)
try:
records = sf.query_all(query)
except SalesforceRefusedRequest as e:
raise PopupException(message=str(e))
format_ = {
"sample":
[list(row.values())[1:] for row in records['records']],
"columns": [
Field(
col['name'],
HiveFormat.FIELD_TYPE_TRANSLATE.get(
col['type'], 'string')).to_dict()
for col in table_metadata
]
}
else:
raise PopupException(
_('Connector format not recognized: %(connectorSelection)s') %
file_format)
else:
raise PopupException(
_('Input format not recognized: %(inputFormat)s') % file_format)
return JsonResponse(format_)
def test_check_status_success(self):
""""
Test method for metadata deployment
"""
# pylint: disable-msg=C0301
mock_response = '<?xml version="1.0" ' \
'encoding="UTF-8"?><soapenv:Envelope ' \
'xmlns:soapenv="http://schemas.xmlsoap.org/soap' \
'/envelope/" ' \
'xmlns="http://soap.sforce.com/2006/04/metadata' \
'"><soapenv:Body><checkDeployStatusResponse><result' \
'><checkOnly>false</checkOnly><completedDate>2020-10' \
'-28T13:33:29.000Z</completedDate><createdBy' \
'>0053D0000052Xaq</createdBy><createdByName>User ' \
'User</createdByName><createdDate>2020-10-28T13:33:25' \
'.000Z</createdDate><details><componentSuccesses' \
'><changed>true</changed><componentType>ApexSettings' \
'</componentType><created>false</created><createdDate' \
'>2020-10-28T13:33:29.000Z</createdDate><deleted' \
'>false</deleted><fileName>shape/settings/Apex' \
'.settings</fileName><fullName>Apex</fullName' \
'><success>true</success></componentSuccesses' \
'><componentSuccesses><changed>true</changed' \
'><componentType>ChatterSettings</componentType' \
'><created>false</created><createdDate>2020-10-28T13' \
':33:29.000Z</createdDate><deleted>false</deleted' \
'><fileName>shape/settings/Chatter.settings</fileName' \
'><fullName>Chatter</fullName><success>true</success' \
'></componentSuccesses><componentSuccesses><changed' \
'>true</changed><componentType></componentType' \
'><created>false</created><createdDate>2020-10-28T13' \
':33:29.000Z</createdDate><deleted>false</deleted' \
'><fileName>shape/package.xml</fileName><fullName' \
'>package.xml</fullName><success>true</success' \
'></componentSuccesses><componentSuccesses><changed' \
'>true</changed><componentType>LightningExperienceSettings</componentType><created>false</created><createdDate>2020-10-28T13:33:29.000Z</createdDate><deleted>false</deleted><fileName>shape/settings/LightningExperience.settings</fileName><fullName>LightningExperience</fullName><success>true</success></componentSuccesses><componentSuccesses><changed>true</changed><componentType>LanguageSettings</componentType><created>false</created><createdDate>2020-10-28T13:33:29.000Z</createdDate><deleted>false</deleted><fileName>shape/settings/Language.settings</fileName><fullName>Language</fullName><success>true</success></componentSuccesses><runTestResult><numFailures>0</numFailures><numTestsRun>0</numTestsRun><totalTime>0.0</totalTime></runTestResult></details><done>true</done><id>0Af3D00001NVCnwSAH</id><ignoreWarnings>false</ignoreWarnings><lastModifiedDate>2020-10-28T13:33:29.000Z</lastModifiedDate><numberComponentErrors>0</numberComponentErrors><numberComponentsDeployed>4</numberComponentsDeployed><numberComponentsTotal>4</numberComponentsTotal><numberTestErrors>0</numberTestErrors><numberTestsCompleted>0</numberTestsCompleted><numberTestsTotal>0</numberTestsTotal><rollbackOnError>true</rollbackOnError><runTestsEnabled>false</runTestsEnabled><startDate>2020-10-28T13:33:26.000Z</startDate><status>Succeeded</status><success>true</success></result></checkDeployStatusResponse></soapenv:Body></soapenv:Envelope>'
responses.add(responses.POST,
re.compile(r'^https://.*/deployRequest/abdcefg'),
body=mock_response,
status=http.OK)
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance=tests.INSTANCE_URL,
session=session)
result = client.checkDeployStatus("abdcefg", sandbox=False)
self.assertEqual(result.get('state'), "Succeeded")
self.assertEqual(result.get('state_detail'), None)
self.assertEqual(
result.get('deployment_detail'), {
'total_count': '4',
'failed_count': '0',
'deployed_count': '4',
'errors': []
})
self.assertEqual(
result.get('unit_test_detail'), {
'total_count': '0',
'failed_count': '0',
'completed_count': '0',
'errors': []
})
def test_query_all(self):
"""Test bulk queryAll records"""
operation = 'queryAll'
responses.add(
responses.POST,
re.compile(r'^https://[^/job].*/job$'),
body='{"apiVersion": 42.0, "concurrencyMode": "Parallel",'
'"contentType": "JSON","id": "Job-1","object": "Contact",'
'"operation": "%s","state": "Open"}' % operation,
status=http.OK)
responses.add(
responses.POST,
re.compile(r'^https://[^/job].*/job/Job-1/batch$'),
body='{"id": "Batch-1","jobId": "Job-1","state": "Queued"}',
status=http.OK)
responses.add(
responses.POST,
re.compile(r'^https://[^/job].*/job/Job-1$'),
body='{"apiVersion" : 42.0, "concurrencyMode" : "Parallel",'
'"contentType" : "JSON","id" : "Job-1","object" : "Contact",'
'"operation" : "%s","state" : "Closed"}' % operation,
status=http.OK)
responses.add(
responses.GET,
re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1$'),
body='{"id": "Batch-1","jobId": "Job-1","state": "InProgress"}',
status=http.OK)
responses.add(
responses.GET,
re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1$'),
body='{"id": "Batch-1","jobId": "Job-1","state": "Completed"}',
status=http.OK)
responses.add(
responses.GET,
re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1/result$'),
body='["752x000000000F1","752x000000000F2"]',
status=http.OK)
responses.add(
responses.GET,
re.compile(
r"""^https://[^/job].*/job/Job-1/batch/Batch-1/result
/752x000000000F1$""", re.X),
body='[{"Id": "001xx000003DHP0AAO", "AccountId": "ID-13",'
'"Email": "*****@*****.**","FirstName": "Bob",'
'"LastName": "x"},{"Id": "001xx000003DHP1AAO",'
'"AccountId": "ID-24","Email": "*****@*****.**",'
'"FirstName": "Alice","LastName": "y"}]',
status=http.OK)
responses.add(
responses.GET,
re.compile(
r"""^https://[^/job].*/job/Job-1/batch/Batch-1/result
/752x000000000F2$""", re.X),
body='[{"Id": "001xx000003DHP0AAO", "AccountId": "ID-13",'
'"Email": "*****@*****.**","FirstName": "Bob",'
'"LastName": "x"},{"Id": "001xx000003DHP1AAO",'
'"AccountId": "ID-24","Email": "*****@*****.**",'
'"FirstName": "Alice","LastName": "y"}]',
status=http.OK)
data = 'SELECT Id,AccountId,Email,FirstName,LastName FROM Contact'
session = requests.Session()
client = Salesforce(session_id=tests.SESSION_ID,
instance_url=tests.SERVER_URL,
session=session)
contact = client.bulk.Contact.query_all(data)
self.assertEqual(self.expected_query, contact)
def guess_format(request):
file_format = json.loads(request.POST.get('fileFormat', '{}'))
file_type = file_format['file_type']
path = urllib_unquote(file_format["path"])
if sys.version_info[0] < 3 and (file_type == 'excel' or path[-3:] == 'xls'
or path[-4:] == 'xlsx'):
return JsonResponse({
'status':
-1,
'message':
'Python2 based Hue does not support Excel file importer'
})
if file_format['inputFormat'] == 'localfile':
if file_type == 'excel':
format_ = {"type": "excel", "hasHeader": True}
else:
format_ = {
"quoteChar": "\"",
"recordSeparator": '\\n',
"type": "csv",
"hasHeader": True,
"fieldSeparator": ","
}
elif file_format['inputFormat'] == 'file':
if path[-3:] == 'xls' or path[-4:] == 'xlsx':
file_obj = request.fs.open(path)
if path[-3:] == 'xls':
df = pd.read_excel(file_obj.read(1024 * 1024 * 1024),
engine='xlrd')
else:
df = pd.read_excel(file_obj.read(1024 * 1024 * 1024),
engine='openpyxl')
_csv_data = df.to_csv(index=False)
path = excel_to_csv_file_name_change(path)
request.fs.create(path, overwrite=True, data=_csv_data)
indexer = MorphlineIndexer(request.user, request.fs)
if not request.fs.isfile(path):
raise PopupException(
_('Path %(path)s is not a file') % file_format)
stream = request.fs.open(path)
format_ = indexer.guess_format(
{"file": {
"stream": stream,
"name": path
}})
_convert_format(format_)
if file_format["path"][-3:] == 'xls' or file_format["path"][
-4:] == 'xlsx':
format_ = {
"quoteChar": "\"",
"recordSeparator": '\\n',
"type": "excel",
"hasHeader": True,
"fieldSeparator": ","
}
elif file_format['inputFormat'] == 'table':
db = dbms.get(request.user)
try:
table_metadata = db.get_table(database=file_format['databaseName'],
table_name=file_format['tableName'])
except Exception as e:
raise PopupException(
e.message if hasattr(e, 'message') and e.message else e)
storage = {}
for delim in table_metadata.storage_details:
if delim['data_type']:
if '=' in delim['data_type']:
key, val = delim['data_type'].split('=', 1)
storage[key] = val
else:
storage[delim['data_type']] = delim['comment']
if table_metadata.details['properties']['format'] == 'text':
format_ = {
"quoteChar": "\"",
"recordSeparator": '\\n',
"type": "csv",
"hasHeader": False,
"fieldSeparator": storage.get('field.delim', ',')
}
elif table_metadata.details['properties']['format'] == 'parquet':
format_ = {
"type": "parquet",
"hasHeader": False,
}
else:
raise PopupException(
'Hive table format %s is not supported.' %
table_metadata.details['properties']['format'])
elif file_format['inputFormat'] == 'query':
format_ = {
"quoteChar": "\"",
"recordSeparator": "\\n",
"type": "csv",
"hasHeader": False,
"fieldSeparator": "\u0001"
}
elif file_format['inputFormat'] == 'rdbms':
format_ = {"type": "csv"}
elif file_format['inputFormat'] == 'stream':
if file_format['streamSelection'] == 'kafka':
format_ = {
"type": "json",
# "fieldSeparator": ",",
# "hasHeader": True,
# "quoteChar": "\"",
# "recordSeparator": "\\n",
'topics': get_topics(request.user)
}
elif file_format['streamSelection'] == 'flume':
format_ = {
"type": "csv",
"fieldSeparator": ",",
"hasHeader": True,
"quoteChar": "\"",
"recordSeparator": "\\n"
}
elif file_format['inputFormat'] == 'connector':
if file_format['connectorSelection'] == 'sfdc':
sf = Salesforce(username=file_format['streamUsername'],
password=file_format['streamPassword'],
security_token=file_format['streamToken'])
format_ = {
"type":
"csv",
"fieldSeparator":
",",
"hasHeader":
True,
"quoteChar":
"\"",
"recordSeparator":
"\\n",
'objects': [
sobject['name']
for sobject in sf.restful('sobjects/')['sobjects']
if sobject['queryable']
]
}
else:
raise PopupException(
_('Input format %(inputFormat)s connector not recognized: $(connectorSelection)s'
) % file_format)
else:
raise PopupException(
_('Input format not recognized: %(inputFormat)s') % file_format)
format_['status'] = 0
return JsonResponse(format_)
def guess_field_types(request):
file_format = json.loads(request.POST.get('fileFormat', '{}'))
if file_format['inputFormat'] == 'file':
indexer = MorphlineIndexer(request.user, request.fs)
path = urllib_unquote(file_format["path"])
stream = request.fs.open(path)
encoding = check_encoding(stream.read(10000))
stream.seek(0)
_convert_format(file_format["format"], inverse=True)
format_ = indexer.guess_field_types({
"file": {
"stream": stream,
"name": path
},
"format": file_format['format']
})
# Note: Would also need to set charset to table (only supported in Hive)
if 'sample' in format_ and format_['sample']:
format_['sample'] = escape_rows(format_['sample'],
nulls_only=True,
encoding=encoding)
for col in format_['columns']:
col['name'] = smart_unicode(col['name'],
errors='replace',
encoding=encoding)
elif file_format['inputFormat'] == 'table':
sample = get_api(request, {
'type': 'hive'
}).get_sample_data({'type': 'hive'},
database=file_format['databaseName'],
table=file_format['tableName'])
db = dbms.get(request.user)
table_metadata = db.get_table(database=file_format['databaseName'],
table_name=file_format['tableName'])
format_ = {
"sample":
sample['rows'][:4],
"columns": [
Field(col.name,
HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type,
'string')).to_dict()
for col in table_metadata.cols
]
}
elif file_format['inputFormat'] == 'query':
query_id = file_format['query']['id'] if file_format['query'].get(
'id') else file_format['query']
notebook = Notebook(document=Document2.objects.document(
user=request.user, doc_id=query_id)).get_data()
snippet = notebook['snippets'][0]
db = get_api(request, snippet)
if file_format.get('sampleCols'):
columns = file_format.get('sampleCols')
sample = file_format.get('sample')
else:
snippet['query'] = snippet['statement']
try:
sample = db.fetch_result(notebook, snippet, 4,
start_over=True)['rows'][:4]
except Exception as e:
LOG.warning(
'Skipping sample data as query handle might be expired: %s'
% e)
sample = [[], [], [], [], []]
columns = db.autocomplete(snippet=snippet, database='', table='')
columns = [
Field(
col['name'],
HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'],
'string')).to_dict()
for col in columns['extended_columns']
]
format_ = {
"sample": sample,
"columns": columns,
}
elif file_format['inputFormat'] == 'rdbms':
api = _get_api(request)
sample = api.get_sample_data(None,
database=file_format['rdbmsDatabaseName'],
table=file_format['tableName'])
format_ = {
"sample":
list(sample['rows'])[:4],
"columns": [
Field(col['name'], col['type']).to_dict()
for col in sample['full_headers']
]
}
elif file_format['inputFormat'] == 'stream':
if file_format['streamSelection'] == 'kafka':
data = get_topic_data(request.user,
file_format.get('kafkaSelectedTopics'))
kafkaFieldNames = [col['name'] for col in data['full_headers']]
kafkaFieldTypes = [col['type'] for col in data['full_headers']]
topics_data = data['rows']
format_ = {
"sample":
topics_data,
"columns": [
Field(col, 'string', unique=False).to_dict()
for col in kafkaFieldNames
]
}
# data = """%(kafkaFieldNames)s
# %(data)s""" % {
# 'kafkaFieldNames': ','.join(kafkaFieldNames),
# 'data': '\n'.join([','.join(cols) for cols in topics_data])
# }
# stream = string_io()
# stream.write(data)
# _convert_format(file_format["format"], inverse=True)
# indexer = MorphlineIndexer(request.user, request.fs)
# format_ = indexer.guess_field_types({
# "file": {
# "stream": stream,
# "name": file_format['path']
# },
# "format": file_format['format']
# })
# type_mapping = dict(
# list(
# zip(kafkaFieldNames, kafkaFieldTypes)
# )
# )
# for col in format_['columns']:
# col['keyType'] = type_mapping[col['name']]
# col['type'] = type_mapping[col['name']]
elif file_format['streamSelection'] == 'flume':
if 'hue-httpd/access_log' in file_format['channelSourcePath']:
columns = [{
'name': 'id',
'type': 'string',
'unique': True
}, {
'name': 'client_ip',
'type': 'string'
}, {
'name': 'time',
'type': 'date'
}, {
'name': 'request',
'type': 'string'
}, {
'name': 'code',
'type': 'plong'
}, {
'name': 'bytes',
'type': 'plong'
}, {
'name': 'method',
'type': 'string'
}, {
'name': 'url',
'type': 'string'
}, {
'name': 'protocol',
'type': 'string'
}, {
'name': 'app',
'type': 'string'
}, {
'name': 'subapp',
'type': 'string'
}]
else:
columns = [{'name': 'message', 'type': 'string'}]
format_ = {
"sample": [['...'] * len(columns)] * 4,
"columns": [
Field(col['name'],
HiveFormat.FIELD_TYPE_TRANSLATE.get(
col['type'], 'string'),
unique=col.get('unique')).to_dict()
for col in columns
]
}
elif file_format['inputFormat'] == 'connector':
if file_format['connectorSelection'] == 'sfdc':
sf = Salesforce(username=file_format['streamUsername'],
password=file_format['streamPassword'],
security_token=file_format['streamToken'])
table_metadata = [{
'name': column['name'],
'type': column['type']
} for column in sf.restful('sobjects/%(streamObject)s/describe/' %
file_format)['fields']]
query = 'SELECT %s FROM %s LIMIT 4' % (', '.join(
[col['name']
for col in table_metadata]), file_format['streamObject'])
print(query)
try:
records = sf.query_all(query)
except SalesforceRefusedRequest as e:
raise PopupException(message=str(e))
format_ = {
"sample":
[list(row.values())[1:] for row in records['records']],
"columns": [
Field(
col['name'],
HiveFormat.FIELD_TYPE_TRANSLATE.get(
col['type'], 'string')).to_dict()
for col in table_metadata
]
}
else:
raise PopupException(
_('Connector format not recognized: %(connectorSelection)s') %
file_format)
else:
raise PopupException(
_('Input format not recognized: %(inputFormat)s') % file_format)
return JsonResponse(format_)
def guess_format(request):
file_format = json.loads(request.POST.get('fileFormat', '{}'))
if file_format['inputFormat'] == 'file':
path = urllib_unquote(file_format["path"])
indexer = MorphlineIndexer(request.user, request.fs)
if not request.fs.isfile(path):
raise PopupException(
_('Path %(path)s is not a file') % file_format)
stream = request.fs.open(path)
format_ = indexer.guess_format(
{"file": {
"stream": stream,
"name": path
}})
_convert_format(format_)
elif file_format['inputFormat'] == 'table':
db = dbms.get(request.user)
try:
table_metadata = db.get_table(database=file_format['databaseName'],
table_name=file_format['tableName'])
except Exception as e:
raise PopupException(
e.message if hasattr(e, 'message') and e.message else e)
storage = {}
for delim in table_metadata.storage_details:
if delim['data_type']:
if '=' in delim['data_type']:
key, val = delim['data_type'].split('=', 1)
storage[key] = val
else:
storage[delim['data_type']] = delim['comment']
if table_metadata.details['properties']['format'] == 'text':
format_ = {
"quoteChar": "\"",
"recordSeparator": '\\n',
"type": "csv",
"hasHeader": False,
"fieldSeparator": storage.get('field.delim', ',')
}
elif table_metadata.details['properties']['format'] == 'parquet':
format_ = {
"type": "parquet",
"hasHeader": False,
}
else:
raise PopupException(
'Hive table format %s is not supported.' %
table_metadata.details['properties']['format'])
elif file_format['inputFormat'] == 'query':
format_ = {
"quoteChar": "\"",
"recordSeparator": "\\n",
"type": "csv",
"hasHeader": False,
"fieldSeparator": "\u0001"
}
elif file_format['inputFormat'] == 'rdbms':
format_ = {"type": "csv"}
elif file_format['inputFormat'] == 'stream':
if file_format['streamSelection'] == 'kafka':
format_ = {
"type": "json",
# "fieldSeparator": ",",
# "hasHeader": True,
# "quoteChar": "\"",
# "recordSeparator": "\\n",
'topics': get_topics(request.user)
}
elif file_format['streamSelection'] == 'flume':
format_ = {
"type": "csv",
"fieldSeparator": ",",
"hasHeader": True,
"quoteChar": "\"",
"recordSeparator": "\\n"
}
elif file_format['inputFormat'] == 'connector':
if file_format['connectorSelection'] == 'sfdc':
sf = Salesforce(username=file_format['streamUsername'],
password=file_format['streamPassword'],
security_token=file_format['streamToken'])
format_ = {
"type":
"csv",
"fieldSeparator":
",",
"hasHeader":
True,
"quoteChar":
"\"",
"recordSeparator":
"\\n",
'objects': [
sobject['name']
for sobject in sf.restful('sobjects/')['sobjects']
if sobject['queryable']
]
}
else:
raise PopupException(
_('Input format %(inputFormat)s connector not recognized: $(connectorSelection)s'
) % file_format)
else:
raise PopupException(
_('Input format not recognized: %(inputFormat)s') % file_format)
format_['status'] = 0
return JsonResponse(format_)
format_ = {"quoteChar": "\"", "recordSeparator": '\\n', "type": "csv", "hasHeader": False, "fieldSeparator": storage.get('field.delim', ',')}
elif table_metadata.details['properties']['format'] == 'parquet':
format_ = {"type": "parquet", "hasHeader": False,}
else:
raise PopupException('Hive table format %s is not supported.' % table_metadata.details['properties']['format'])
elif file_format['inputFormat'] == 'query':
format_ = {"quoteChar": "\"", "recordSeparator": "\\n", "type": "csv", "hasHeader": False, "fieldSeparator": "\u0001"}
elif file_format['inputFormat'] == 'rdbms':
format_ = RdbmsIndexer(request.user, file_format['rdbmsType']).guess_format()
elif file_format['inputFormat'] == 'stream':
if file_format['streamSelection'] == 'kafka':
format_ = {"type": "csv", "fieldSeparator": ",", "hasHeader": True, "quoteChar": "\"", "recordSeparator": "\\n", 'topics': get_topics()}
elif file_format['streamSelection'] == 'sfdc':
sf = Salesforce(
username=file_format['streamUsername'],
password=file_format['streamPassword'],
security_token=file_format['streamToken']
)
format_ = {"type": "csv", "fieldSeparator": ",", "hasHeader": True, "quoteChar": "\"", "recordSeparator": "\\n", 'objects': [sobject['name'] for sobject in sf.restful('sobjects/')['sobjects'] if sobject['queryable']]}
format_['status'] = 0
return JsonResponse(format_)
def guess_field_types(request):
file_format = json.loads(request.POST.get('fileFormat', '{}'))
if file_format['inputFormat'] == 'file':
indexer = MorphlineIndexer(request.user, request.fs)
path = urllib.unquote(file_format["path"])
stream = request.fs.open(path)
elif file_format['inputFormat'] == 'rdbms':
format_ = RdbmsIndexer(request.user,
file_format['rdbmsType']).guess_format()
elif file_format['inputFormat'] == 'stream':
if file_format['streamSelection'] == 'kafka':
format_ = {
"type": "csv",
"fieldSeparator": ",",
"hasHeader": True,
"quoteChar": "\"",
"recordSeparator": "\\n",
'topics': get_topics()
}
elif file_format['streamSelection'] == 'sfdc':
sf = Salesforce(username=file_format['streamUsername'],
password=file_format['streamPassword'],
security_token=file_format['streamToken'])
format_ = {
"type":
"csv",
"fieldSeparator":
",",
"hasHeader":
True,
"quoteChar":
"\"",
"recordSeparator":
"\\n",
'objects': [
sobject['name']
for sobject in sf.restful('sobjects/')['sobjects']
def __init__(self, username, password, token):
'''
Constructor
'''
self.api = Salesforce(username=username, password=password, security_token=token)
super(SalesforceExtractor, self).__init__(self._incoming_table_class)
