def test_query_all_include_deleted(self): """ Test that we query all and fetch additional result sets automatically. """ responses.add( responses.GET, re.compile(r'^https://.*/queryAll/\?q=SELECT\+ID\+FROM\+Account$'), body='{"records": [{"ID": "1"}], "done": false, "nextRecordsUrl": ' '"https://example.com/queryAll/next-records-id"}', status=http.OK) responses.add( responses.GET, re.compile(r'^https://.*/queryAll/next-records-id$'), body='{"records": [{"ID": "2"}], "done": true}', status=http.OK) session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session) result = client.query_all('SELECT ID FROM Account', include_deleted=True) self.assertEqual( result, OrderedDict([(u'records', [ OrderedDict([(u'ID', u'1')]), OrderedDict([(u'ID', u'2')]) ]), (u'done', True)]))
def test_query_include_deleted(self): """Test querying for all records generates the expected request""" responses.add( responses.GET, re.compile(r'^https://.*/queryAll/\?q=SELECT\+ID\+FROM\+Account$'), body='{}', status=http.OK) session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session) result = client.query('SELECT ID FROM Account', include_deleted=True) self.assertEqual(result, {})
def test_query_more_id_not_url(self): """ Test fetching additional results by ID generates the expected request """ responses.add( responses.GET, re.compile(r'^https://.*/query/next-records-id$'), body='{}', status=http.OK) session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session) result = client.query_more('next-records-id', identifier_is_url=False) self.assertEqual(result, {})
def test_api_limits(self): """Test method for getting Salesforce organization limits""" responses.add( responses.GET, re.compile(r'^https://.*/limits/$'), json=tests.ORGANIZATION_LIMITS_RESPONSE, status=http.OK ) session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session) result = client.limits() self.assertEqual(result, tests.ORGANIZATION_LIMITS_RESPONSE)
def test_proxies_inherited_set_on_session(self): """Test Salesforce and SFType use same custom proxies""" session = requests.Session() session.proxies = tests.PROXIES client = Salesforce(session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session) self.assertIs(tests.PROXIES, client.session.proxies) self.assertIs(tests.PROXIES, client.Contact.session.proxies)
def test_proxies_inherited_default(self): """Test Salesforce and SFType use same proxies""" session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session) self.assertIs(session.proxies, client.session.proxies) self.assertIs(session.proxies, client.Contact.session.proxies)
def test_shared_custom_session_to_sftype(self): """Test Salesforce and SFType instances share custom `Session`""" session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session) self.assertIs(session, client.session) self.assertIs(session, client.Contact.session)
def test_md_deploy_failed_status_code(self, mock_read_zip): """" Test method for metadata deployment """ #pylint: disable=unused-argument responses.add( responses.POST, re.compile(r'^https://.*'), body="Unrecognized Error", status=2599 ) session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance=tests.INSTANCE_URL, session=session) with self.assertRaises(Exception): client.deploy("path/to/fake/zip.zip", sandbox=False)
def test_query_parse_float(self): """Test parsing of floating point numbers as floats""" responses.add( responses.GET, re.compile(r'^https://.*/query/\?q=SELECT\+ID%2C\+Price\+' r'FROM\+Account$'), body='{"records": [{"ID": "1", "Price": 13.40},' '{"ID": "2", "Price": 1.12345678901234567},' '{"ID": "3", "Price": 123456789012345678},' '{"ID": "4", "Price": 0},' '{"ID": "5", "Price": -1.1234567890123456},' '{"ID": "6", "Price": -12345678901234567}],' '"done": false, "nextRecordsUrl": ' '"https://example.com/query/next-records-id", "totalSize": 6}', status=http.OK) session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session) result = client.query('SELECT ID, Price FROM Account') self.assertEqual( result, OrderedDict([ ('records', [ OrderedDict([('ID', '1'), ('Price', 13.4)]), OrderedDict([ ('ID', '2'), ('Price', 1.12345678901234567) ]), OrderedDict([ ('ID', '3'), ('Price', 123456789012345678) ]), OrderedDict([('ID', '4'), ('Price', 0)]), OrderedDict([ ('ID', '5'), ('Price', -1.1234567890123456), ]), OrderedDict([ ('ID', '6'), ('Price', -12345678901234567), ]), ]), ('done', False), ('nextRecordsUrl', "https://example.com/query/next-records-id"), ('totalSize', 6), ]))
def test_delete(self): """Test delete records""" operation = 'delete' responses.add( responses.POST, re.compile(r'^https://[^/job].*/job$'), body='{"apiVersion": 42.0, "concurrencyMode": "Parallel",' '"contentType": "JSON","id": "Job-1","object": "Contact",' '"operation": "%s","state": "Open"}' % operation, status=http.OK) responses.add( responses.POST, re.compile(r'^https://[^/job].*/job/Job-1/batch$'), body='{"id": "Batch-1","jobId": "Job-1","state": "Queued"}', status=http.OK ) responses.add( responses.POST, re.compile(r'^https://[^/job].*/job/Job-1$'), body='{"apiVersion" : 42.0, "concurrencyMode" : "Parallel",' '"contentType" : "JSON","id" : "Job-1","object" : "Contact",' '"operation" : "%s","state" : "Closed"}' % operation, status=http.OK ) responses.add( responses.GET, re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1$'), body='{"id": "Batch-1","jobId": "Job-1","state": "InProgress"}', status=http.OK ) responses.add( responses.GET, re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1$'), body='{"id": "Batch-1","jobId": "Job-1","state": "Completed"}', status=http.OK ) responses.add( responses.GET, re.compile( r'^https://[^/job].*/job/Job-1/batch/Batch-1/result$'), body='[{"success": true,"created": true,"id": "001xx000003DHP0AAO",' '"errors": []},{"success": true,"created": true,' '"id": "001xx000003DHP1AAO","errors": []}]', status=http.OK ) data = [{ 'id': 'ID-1', }, { 'id': 'ID-2', }] session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session) contact = client.bulk.Contact.delete(data) self.assertEqual(self.expected, contact)
def test_proxies_ignored(self): """Test overridden proxies are ignored""" session = requests.Session() session.proxies = tests.PROXIES with patch('simple_salesforce.api.logger.warning') as mock_log: client = Salesforce(session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session, proxies={}) self.assertIn('ignoring proxies', mock_log.call_args[0][0]) self.assertIs(tests.PROXIES, client.session.proxies)
def test_bulk_type(self): """Test bulk type creation""" session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session) contact = client.bulk.Contact self.assertIs('Contact', contact.object_name) self.assertIs(client.bulk_url, contact.bulk_url) self.assertEqual(tests.BULK_HEADERS, contact.headers)
def test_bulk_handler(self): """Test bulk handler creation""" session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session) bulk_handler = client.bulk self.assertIs(tests.SESSION_ID, bulk_handler.session_id) self.assertIs(client.bulk_url, bulk_handler.bulk_url) self.assertEqual(tests.BULK_HEADERS, bulk_handler.headers)
def test_check_status_payload_error(self, mock_post): """ " Test method for metadata deployment """ mock_response = Mock() mock_response.status_code = 200 mock_response.text = '<?xml version="1.0" encoding="UTF-8"?><soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns="http://soap.sforce.com/2006/04/metadata"><soapenv:Body><checkDeployStatusResponse><result><checkOnly>true</checkOnly><completedDate>2020-10-28T13:37:48.000Z</completedDate><createdBy>0053D0000052Xaq</createdBy><createdByName>User User</createdByName><createdDate>2020-10-28T13:37:46.000Z</createdDate><details><componentFailures><changed>false</changed><componentType></componentType><created>false</created><createdDate>2020-10-28T13:37:47.000Z</createdDate><deleted>false</deleted><fileName>package.xml</fileName><fullName>package.xml</fullName><problem>No package.xml found</problem><problemType>Error</problemType><success>false</success></componentFailures><runTestResult><numFailures>0</numFailures><numTestsRun>0</numTestsRun><totalTime>0.0</totalTime></runTestResult></details><done>true</done><id>0Af3D00001NVD0TSAX</id><ignoreWarnings>false</ignoreWarnings><lastModifiedDate>2020-10-28T13:37:48.000Z</lastModifiedDate><numberComponentErrors>0</numberComponentErrors><numberComponentsDeployed>0</numberComponentsDeployed><numberComponentsTotal>0</numberComponentsTotal><numberTestErrors>0</numberTestErrors><numberTestsCompleted>0</numberTestsCompleted><numberTestsTotal>0</numberTestsTotal><rollbackOnError>true</rollbackOnError><runTestsEnabled>false</runTestsEnabled><startDate>2020-10-28T13:37:47.000Z</startDate><status>Failed</status><success>false</success></result></checkDeployStatusResponse></soapenv:Body></soapenv:Envelope>' mock_post.return_value = mock_response session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance=tests.SERVER_URL, session=session) state, state_detail, deployment_detail, unit_test_detail = client.checkDeployStatus( "abdcefg") self.assertEqual(state, "Failed") self.assertEqual(state_detail, None) self.assertEqual( deployment_detail, { "total_count": "0", "failed_count": "0", "deployed_count": "0", "errors": [{ "type": None, "file": "package.xml", "status": "Error", "message": "No package.xml found" }], }, ) self.assertEqual( unit_test_detail, { "total_count": "0", "failed_count": "0", "completed_count": "0", "errors": [] })
def test_query_more_parse_float_to_decimal(self): """Test querying generates float as Decimal values""" responses.add( responses.GET, re.compile(r'^https://.*/query/next-records-id$'), body='{"currency": 1.0}', status=http.OK, ) session = requests.Session() client = Salesforce( session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session, parse_float=decimal.Decimal, ) result = client.query_more('next-records-id', identifier_is_url=False) self.assertIsInstance(result["currency"], decimal.Decimal) self.assertNotIsInstance(result["currency"], float) self.assertEqual(result, {"currency": decimal.Decimal(1.0)}) self.assertEqual(result, {"currency": 1.0}) self.assertNotEqual(result, {"currency": "1.0"})
class SalesforceExtractor(Extractor): _incoming_table_class = IncomingSalesforceRecord def __init__(self, username, password, token): ''' Constructor ''' self.api = Salesforce(username=username, password=password, security_token=token) super(SalesforceExtractor, self).__init__(self._incoming_table_class) def do_extract(self): res = self.api.describe() obj_names = [obj['name'] for obj in res['sobjects']] obj_fields = dict() for obj in obj_names: api_method_call = getattr(self.api, obj) try: obj_meta = api_method_call.describe() # get field names for everything except the content blobs # if there's a blob in the record queries will return only one at a time i.e. one API call per record field_names = [field['name'] for field in obj_meta['fields'] if field['type'] not in ('base64')] obj_fields[obj] = field_names except SalesforceMalformedRequest, m: logger.warn('Got SalesforceMalformedRequest when querying metadata for object %s' % obj) logger.warn(str(m)) for obj in obj_fields.keys(): fieldstr = ','.join(obj_fields[obj]) querystr = 'SELECT %s FROM %s' % (fieldstr,obj) logger.info('Executing Salesforce query %s' % querystr) records = [] try: obj_res = self.api.query(querystr) if 'records' in obj_res: records.extend(obj_res['records']) while 'done' in obj_res and not obj_res['done']: obj_res = self.api.query_more(obj_res['nextRecordsUrl'], True) if 'records' in obj_res: records.extend(obj_res['records']) logger.info('%d results of type %s' % (len(records),obj)) for rec in records: IncomingSalesforceRecord.create(sf_id=id,object_type=obj,record=json.dumps(rec)) except SalesforceMalformedRequest, m: logger.warn('Got SalesforceMalformedRequest when trying to retrieve all fields for object %s' % obj) logger.warn(str(m))
def test_query_parse_float_to_decimal(self): """Test querying generates float as Decimal values""" responses.add( responses.GET, re.compile( r'^https://.*/query/\?q=SELECT\+currency\+FROM\+Account$' ), body='{"currency": 1.0}', status=http.OK, ) session = requests.Session() client = Salesforce( session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session, parse_float=decimal.Decimal, ) result = client.query('SELECT currency FROM Account') self.assertIsInstance(result["currency"], decimal.Decimal) self.assertNotIsInstance(result["currency"], float) self.assertEqual(result, {"currency": decimal.Decimal(1.0)}) self.assertEqual(result, {"currency": 1.0}) self.assertNotEqual(result, {"currency": "1.0"})
def test_api_usage_simple(self): """Make sure a header response is recorded""" responses.add( responses.GET, re.compile(r'^https://.*$'), body='{"example": 1}', adding_headers={"Sforce-Limit-Info": "api-usage=18/5000"}, status=http.OK) client = Salesforce.__new__(Salesforce) client.request = requests.Session() client.headers = {} client.base_url = 'https://localhost' client.query('q') self.assertDictEqual(client.api_usage, {'api-usage': Usage(18, 5000)})
def test_api_usage_simple(self): """Make sure a header response is recorded""" responses.add( responses.GET, re.compile(r'^https://.*$'), body='{"example": 1}', adding_headers={"Sforce-Limit-Info": "api-usage=18/5000"}, status=http.OK ) client = Salesforce.__new__(Salesforce) client.request = requests.Session() client.headers = {} client.base_url = 'https://localhost' client.query('q') self.assertDictEqual(client.api_usage, {'api-usage': Usage(18, 5000)})
def test_custom_version_success(self): """Test custom version""" responses.add(responses.POST, re.compile(r'^https://.*$'), body=tests.LOGIN_RESPONSE_SUCCESS, status=http.OK) # Use an invalid version that is guaranteed to never be used expected_version = '4.2' client = Salesforce(session=requests.Session(), username='******', password='******', security_token='token', version=expected_version) self.assertEqual( client.base_url.split('/')[-2], 'v%s' % expected_version)
def test_custom_version_success(self): """Test custom version""" responses.add(responses.POST, re.compile(r"^https://.*$"), body=tests.LOGIN_RESPONSE_SUCCESS, status=http.OK) # Use an invalid version that is guaranteed to never be used expected_version = "4.2" client = Salesforce( session=requests.Session(), username="******", password="******", security_token="token", version=expected_version, ) self.assertEqual( client.base_url.split("/")[-2], "v%s" % expected_version)
def test_query_fail(self): """Test bulk query records failure""" operation = 'query' responses.add( responses.POST, re.compile(r'^https://[^/job].*/job$'), body='{"apiVersion": 42.0, "concurrencyMode": "Parallel",' '"contentType": "JSON","id": "Job-1","object": "Contact",' '"operation": "%s","state": "Open"}' % operation, status=http.OK) responses.add( responses.POST, re.compile(r'^https://[^/job].*/job/Job-1/batch$'), body='{"id": "Batch-1","jobId": "Job-1","state": "Queued"}', status=http.OK) responses.add( responses.POST, re.compile(r'^https://[^/job].*/job/Job-1$'), body='{"apiVersion" : 42.0, "concurrencyMode" : "Parallel",' '"contentType" : "JSON","id" : "Job-1","object" : "Contact",' '"operation" : "%s","state" : "Closed"}' % operation, status=http.OK) responses.add( responses.GET, re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1$'), body='{"id": "Batch-1","jobId": "Job-1","state": "InProgress"}', status=http.OK) responses.add( responses.GET, re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1$'), body='{"id": "Batch-1","jobId": "Job-1","state": "Failed",' '"stateMessage": "InvalidBatch : Failed to process query"}', status=http.OK) data = 'SELECT ASDFASfgsds FROM Contact' session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session) self.assertRaises(SalesforceGeneralError, client.bulk.Contact.query, data)
def test_api_usage_per_app(self): """Make sure a header response is recorded""" pau = "api-usage=25/5000; per-app-api-usage=17/250(appName=sample-app)" responses.add(responses.GET, re.compile(r'^https://.*$'), body='{"example": 1}', adding_headers={"Sforce-Limit-Info": pau}, status=http.OK) client = Salesforce.__new__(Salesforce) client.request = requests.Session() client.headers = {} client.base_url = 'https://localhost' client.query('q') self.assertDictEqual( client.api_usage, { 'api-usage': Usage(25, 5000), 'per-app-api-usage': PerAppUsage(17, 250, 'sample-app') })
def test_api_usage_per_app(self): """Make sure a header response is recorded""" pau = "api-usage=25/5000; per-app-api-usage=17/250(appName=sample-app)" responses.add( responses.GET, re.compile(r'^https://.*$'), body='{"example": 1}', adding_headers={"Sforce-Limit-Info": pau}, status=http.OK ) client = Salesforce.__new__(Salesforce) client.request = requests.Session() client.headers = {} client.base_url = 'https://localhost' client.query('q') self.assertDictEqual(client.api_usage, {'api-usage': Usage(25, 5000), 'per-app-api-usage': PerAppUsage(17, 250, 'sample-app')})
def test_custom_session_success(self): httpretty.register_uri(httpretty.POST, re.compile(r'^https://.*$'), body=tests.LOGIN_RESPONSE_SUCCESS, status=http.OK) session_state = { 'called': False, } def on_response(*args, **kwargs): session_state['called'] = True session = requests.Session() session.hooks = { 'response': on_response, } sf = Salesforce(session=session, username='******', password='******', security_token='token') self.assertEqual(tests.SESSION_ID, sf.session_id) self.assertEqual(session, sf.request)
def test_check_status_payload_error(self): """" Test method for metadata deployment """ # pylint: disable-msg=C0301 mock_response = '<?xml version="1.0" ' \ 'encoding="UTF-8"?><soapenv:Envelope ' \ 'xmlns:soapenv="http://schemas.xmlsoap.org/soap' \ '/envelope/" ' \ 'xmlns="http://soap.sforce.com/2006/04/metadata' \ '"><soapenv:Body><checkDeployStatusResponse><result' \ '><checkOnly>true</checkOnly><completedDate>2020-10' \ '-28T13:37:48.000Z</completedDate><createdBy' \ '>0053D0000052Xaq</createdBy><createdByName>User ' \ 'User</createdByName><createdDate>2020-10-28T13:37:46' \ '.000Z</createdDate><details><componentFailures' \ '><changed>false</changed><componentType' \ '></componentType><created>false</created' \ '><createdDate>2020-10-28T13:37:47.000Z</createdDate' \ '><deleted>false</deleted><fileName>package.xml' \ '</fileName><fullName>package.xml</fullName><problem' \ '>No package.xml ' \ 'found</problem><problemType>Error</problemType' \ '><success>false</success></componentFailures' \ '><runTestResult><numFailures>0</numFailures' \ '><numTestsRun>0</numTestsRun><totalTime>0.0' \ '</totalTime></runTestResult></details><done>true' \ '</done><id>0Af3D00001NVD0TSAX</id><ignoreWarnings' \ '>false</ignoreWarnings><lastModifiedDate>2020-10' \ '-28T13:37:48.000Z</lastModifiedDate' \ '><numberComponentErrors>0</numberComponentErrors' \ '><numberComponentsDeployed>0</numberComponentsDeployed><numberComponentsTotal>0</numberComponentsTotal><numberTestErrors>0</numberTestErrors><numberTestsCompleted>0</numberTestsCompleted><numberTestsTotal>0</numberTestsTotal><rollbackOnError>true</rollbackOnError><runTestsEnabled>false</runTestsEnabled><startDate>2020-10-28T13:37:47.000Z</startDate><status>Failed</status><success>false</success></result></checkDeployStatusResponse></soapenv:Body></soapenv:Envelope>' responses.add(responses.POST, re.compile(r'^https://.*/deployRequest/abdcefg'), body=mock_response, status=http.OK) session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance=tests.INSTANCE_URL, session=session) result = client.checkDeployStatus("abdcefg", sandbox=False) self.assertEqual(result.get('state'), "Failed") self.assertEqual(result.get('state_detail'), None) self.assertEqual( result.get('deployment_detail'), { 'total_count': '0', 'failed_count': '0', 'deployed_count': '0', 'errors': [{ 'type': None, 'file': 'package.xml', 'status': 'Error', 'message': 'No package.xml found' }] }) self.assertEqual( result.get('unit_test_detail'), { 'total_count': '0', 'failed_count': '0', 'completed_count': '0', 'errors': [] })
def test_upsert(self): """Test upsert records""" operation = 'upsert' responses.add( responses.POST, re.compile(r'^https://[^/job].*/job$'), body='{"apiVersion": 42.0, "concurrencyMode": "Parallel",' '"contentType": "JSON","id": "Job-1","object": "Contact",' '"operation": "%s","state": "Open"}' % operation, status=http.OK) responses.add( responses.POST, re.compile(r'^https://[^/job].*/job/Job-1/batch$'), body='{"id": "Batch-1","jobId": "Job-1","state": "Queued"}', status=http.OK ) responses.add( responses.POST, re.compile(r'^https://[^/job].*/job/Job-1$'), body='{"apiVersion" : 42.0, "concurrencyMode" : "Parallel",' '"contentType" : "JSON","id" : "Job-1","object" : "Contact",' '"operation" : "%s","state" : "Closed"}' % operation, status=http.OK ) responses.add( responses.GET, re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1$'), body='{"id": "Batch-1","jobId": "Job-1","state": "InProgress"}', status=http.OK ) responses.add( responses.GET, re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1$'), body='{"id": "Batch-1","jobId": "Job-1","state": "Completed"}', status=http.OK ) responses.add( responses.GET, re.compile( r'^https://[^/job].*/job/Job-1/batch/Batch-1/result$'), body='[{"success": true,"created": true,"id": "001xx000003DHP0AAO",' '"errors": []},{"success": true,"created": true,' '"id": "001xx000003DHP1AAO","errors": []}]', status=http.OK ) data = [{ 'Custom_Id__c': 'CustomID1', 'AccountId': 'ID-13', 'Email': '*****@*****.**', 'FirstName': 'Bob', 'LastName': 'x' }, { 'Custom_Id__c': 'CustomID2', 'AccountId': 'ID-24', 'Email': '*****@*****.**', 'FirstName': 'Alice', 'LastName': 'y' }] session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session) contact = client.bulk.Contact.upsert(data, 'Custom_Id__c') self.assertEqual(self.expected, contact)
def test_shared_session_to_sftype(self): """Test Salesforce and SFType instances share default `Session`""" client = Salesforce(session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL) self.assertIs(client.session, client.Contact.session)
def guess_field_types(request): file_format = json.loads(request.POST.get('fileFormat', '{}')) if file_format['inputFormat'] == 'file': indexer = MorphlineIndexer(request.user, request.fs) path = urllib_unquote(file_format["path"]) stream = request.fs.open(path) encoding = chardet.detect(stream.read(10000)).get('encoding') stream.seek(0) _convert_format(file_format["format"], inverse=True) format_ = indexer.guess_field_types({ "file": { "stream": stream, "name": path }, "format": file_format['format'] }) # Note: Would also need to set charset to table (only supported in Hive) if 'sample' in format_ and format_['sample']: format_['sample'] = escape_rows(format_['sample'], nulls_only=True, encoding=encoding) for col in format_['columns']: col['name'] = smart_unicode(col['name'], errors='replace', encoding=encoding) elif file_format['inputFormat'] == 'table': sample = get_api(request, { 'type': 'hive' }).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName']) db = dbms.get(request.user) table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName']) format_ = { "sample": sample['rows'][:4], "columns": [ Field(col.name, HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type, 'string')).to_dict() for col in table_metadata.cols ] } elif file_format['inputFormat'] == 'query': query_id = file_format['query']['id'] if file_format['query'].get( 'id') else file_format['query'] notebook = Notebook(document=Document2.objects.document( user=request.user, doc_id=query_id)).get_data() snippet = notebook['snippets'][0] db = get_api(request, snippet) if file_format.get('sampleCols'): columns = file_format.get('sampleCols') sample = file_format.get('sample') else: snippet['query'] = snippet['statement'] try: sample = db.fetch_result(notebook, snippet, 4, start_over=True)['rows'][:4] except Exception as e: LOG.warn( 'Skipping sample data as query handle might be expired: %s' % e) sample = [[], [], [], [], []] columns = db.autocomplete(snippet=snippet, database='', table='') columns = [ Field( col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict() for col in columns['extended_columns'] ] format_ = { "sample": sample, "columns": columns, } elif file_format['inputFormat'] == 'rdbms': api = _get_api(request) sample = api.get_sample_data(None, database=file_format['rdbmsDatabaseName'], table=file_format['tableName']) format_ = { "sample": list(sample['rows'])[:4], "columns": [ Field(col['name'], col['type']).to_dict() for col in sample['full_headers'] ] } elif file_format['inputFormat'] == 'stream': if file_format['streamSelection'] == 'kafka': if file_format.get( 'kafkaSelectedTopics') == 'NavigatorAuditEvents': kafkaFieldNames = [ 'id', 'additionalInfo', 'allowed', 'collectionName', 'databaseName', 'db', 'DELEGATION_TOKEN_ID', 'dst', 'entityId', 'family', 'impersonator', 'ip', 'name', 'objectType', 'objType', 'objUsageType', 'operationParams', 'operationText', 'op', 'opText', 'path', 'perms', 'privilege', 'qualifier', 'QUERY_ID', 'resourcePath', 'service', 'SESSION_ID', 'solrVersion', 'src', 'status', 'subOperation', 'tableName', 'table', 'time', 'type', 'url', 'user' ] kafkaFieldTypes = ['string'] * len(kafkaFieldNames) kafkaFieldNames.append('timeDate') kafkaFieldTypes.append('date') else: # Note: mocked here, should come from SFDC or Kafka API or sampling job kafkaFieldNames = file_format.get('kafkaFieldNames', '').split(',') kafkaFieldTypes = file_format.get('kafkaFieldTypes', '').split(',') data = """%(kafkaFieldNames)s %(data)s""" % { 'kafkaFieldNames': ','.join(kafkaFieldNames), 'data': '\n'.join( [','.join(['...'] * len(kafkaFieldTypes))] * 5) } stream = string_io() stream.write(data) _convert_format(file_format["format"], inverse=True) indexer = MorphlineIndexer(request.user, request.fs) format_ = indexer.guess_field_types({ "file": { "stream": stream, "name": file_format['path'] }, "format": file_format['format'] }) type_mapping = dict(list(zip(kafkaFieldNames, kafkaFieldTypes))) for col in format_['columns']: col['keyType'] = type_mapping[col['name']] col['type'] = type_mapping[col['name']] elif file_format['streamSelection'] == 'flume': if 'hue-httpd/access_log' in file_format['channelSourcePath']: columns = [{ 'name': 'id', 'type': 'string', 'unique': True }, { 'name': 'client_ip', 'type': 'string' }, { 'name': 'time', 'type': 'date' }, { 'name': 'request', 'type': 'string' }, { 'name': 'code', 'type': 'plong' }, { 'name': 'bytes', 'type': 'plong' }, { 'name': 'method', 'type': 'string' }, { 'name': 'url', 'type': 'string' }, { 'name': 'protocol', 'type': 'string' }, { 'name': 'app', 'type': 'string' }, { 'name': 'subapp', 'type': 'string' }] else: columns = [{'name': 'message', 'type': 'string'}] format_ = { "sample": [['...'] * len(columns)] * 4, "columns": [ Field(col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get( col['type'], 'string'), unique=col.get('unique')).to_dict() for col in columns ] } elif file_format['inputFormat'] == 'connector': if file_format['connectorSelection'] == 'sfdc': sf = Salesforce(username=file_format['streamUsername'], password=file_format['streamPassword'], security_token=file_format['streamToken']) table_metadata = [{ 'name': column['name'], 'type': column['type'] } for column in sf.restful('sobjects/%(streamObject)s/describe/' % file_format)['fields']] query = 'SELECT %s FROM %s LIMIT 4' % (', '.join( [col['name'] for col in table_metadata]), file_format['streamObject']) print(query) try: records = sf.query_all(query) except SalesforceRefusedRequest as e: raise PopupException(message=str(e)) format_ = { "sample": [list(row.values())[1:] for row in records['records']], "columns": [ Field( col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get( col['type'], 'string')).to_dict() for col in table_metadata ] } else: raise PopupException( _('Connector format not recognized: %(connectorSelection)s') % file_format) else: raise PopupException( _('Input format not recognized: %(inputFormat)s') % file_format) return JsonResponse(format_)
def test_check_status_success(self): """" Test method for metadata deployment """ # pylint: disable-msg=C0301 mock_response = '<?xml version="1.0" ' \ 'encoding="UTF-8"?><soapenv:Envelope ' \ 'xmlns:soapenv="http://schemas.xmlsoap.org/soap' \ '/envelope/" ' \ 'xmlns="http://soap.sforce.com/2006/04/metadata' \ '"><soapenv:Body><checkDeployStatusResponse><result' \ '><checkOnly>false</checkOnly><completedDate>2020-10' \ '-28T13:33:29.000Z</completedDate><createdBy' \ '>0053D0000052Xaq</createdBy><createdByName>User ' \ 'User</createdByName><createdDate>2020-10-28T13:33:25' \ '.000Z</createdDate><details><componentSuccesses' \ '><changed>true</changed><componentType>ApexSettings' \ '</componentType><created>false</created><createdDate' \ '>2020-10-28T13:33:29.000Z</createdDate><deleted' \ '>false</deleted><fileName>shape/settings/Apex' \ '.settings</fileName><fullName>Apex</fullName' \ '><success>true</success></componentSuccesses' \ '><componentSuccesses><changed>true</changed' \ '><componentType>ChatterSettings</componentType' \ '><created>false</created><createdDate>2020-10-28T13' \ ':33:29.000Z</createdDate><deleted>false</deleted' \ '><fileName>shape/settings/Chatter.settings</fileName' \ '><fullName>Chatter</fullName><success>true</success' \ '></componentSuccesses><componentSuccesses><changed' \ '>true</changed><componentType></componentType' \ '><created>false</created><createdDate>2020-10-28T13' \ ':33:29.000Z</createdDate><deleted>false</deleted' \ '><fileName>shape/package.xml</fileName><fullName' \ '>package.xml</fullName><success>true</success' \ '></componentSuccesses><componentSuccesses><changed' \ '>true</changed><componentType>LightningExperienceSettings</componentType><created>false</created><createdDate>2020-10-28T13:33:29.000Z</createdDate><deleted>false</deleted><fileName>shape/settings/LightningExperience.settings</fileName><fullName>LightningExperience</fullName><success>true</success></componentSuccesses><componentSuccesses><changed>true</changed><componentType>LanguageSettings</componentType><created>false</created><createdDate>2020-10-28T13:33:29.000Z</createdDate><deleted>false</deleted><fileName>shape/settings/Language.settings</fileName><fullName>Language</fullName><success>true</success></componentSuccesses><runTestResult><numFailures>0</numFailures><numTestsRun>0</numTestsRun><totalTime>0.0</totalTime></runTestResult></details><done>true</done><id>0Af3D00001NVCnwSAH</id><ignoreWarnings>false</ignoreWarnings><lastModifiedDate>2020-10-28T13:33:29.000Z</lastModifiedDate><numberComponentErrors>0</numberComponentErrors><numberComponentsDeployed>4</numberComponentsDeployed><numberComponentsTotal>4</numberComponentsTotal><numberTestErrors>0</numberTestErrors><numberTestsCompleted>0</numberTestsCompleted><numberTestsTotal>0</numberTestsTotal><rollbackOnError>true</rollbackOnError><runTestsEnabled>false</runTestsEnabled><startDate>2020-10-28T13:33:26.000Z</startDate><status>Succeeded</status><success>true</success></result></checkDeployStatusResponse></soapenv:Body></soapenv:Envelope>' responses.add(responses.POST, re.compile(r'^https://.*/deployRequest/abdcefg'), body=mock_response, status=http.OK) session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance=tests.INSTANCE_URL, session=session) result = client.checkDeployStatus("abdcefg", sandbox=False) self.assertEqual(result.get('state'), "Succeeded") self.assertEqual(result.get('state_detail'), None) self.assertEqual( result.get('deployment_detail'), { 'total_count': '4', 'failed_count': '0', 'deployed_count': '4', 'errors': [] }) self.assertEqual( result.get('unit_test_detail'), { 'total_count': '0', 'failed_count': '0', 'completed_count': '0', 'errors': [] })
def test_query_all(self): """Test bulk queryAll records""" operation = 'queryAll' responses.add( responses.POST, re.compile(r'^https://[^/job].*/job$'), body='{"apiVersion": 42.0, "concurrencyMode": "Parallel",' '"contentType": "JSON","id": "Job-1","object": "Contact",' '"operation": "%s","state": "Open"}' % operation, status=http.OK) responses.add( responses.POST, re.compile(r'^https://[^/job].*/job/Job-1/batch$'), body='{"id": "Batch-1","jobId": "Job-1","state": "Queued"}', status=http.OK) responses.add( responses.POST, re.compile(r'^https://[^/job].*/job/Job-1$'), body='{"apiVersion" : 42.0, "concurrencyMode" : "Parallel",' '"contentType" : "JSON","id" : "Job-1","object" : "Contact",' '"operation" : "%s","state" : "Closed"}' % operation, status=http.OK) responses.add( responses.GET, re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1$'), body='{"id": "Batch-1","jobId": "Job-1","state": "InProgress"}', status=http.OK) responses.add( responses.GET, re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1$'), body='{"id": "Batch-1","jobId": "Job-1","state": "Completed"}', status=http.OK) responses.add( responses.GET, re.compile(r'^https://[^/job].*/job/Job-1/batch/Batch-1/result$'), body='["752x000000000F1","752x000000000F2"]', status=http.OK) responses.add( responses.GET, re.compile( r"""^https://[^/job].*/job/Job-1/batch/Batch-1/result /752x000000000F1$""", re.X), body='[{"Id": "001xx000003DHP0AAO", "AccountId": "ID-13",' '"Email": "*****@*****.**","FirstName": "Bob",' '"LastName": "x"},{"Id": "001xx000003DHP1AAO",' '"AccountId": "ID-24","Email": "*****@*****.**",' '"FirstName": "Alice","LastName": "y"}]', status=http.OK) responses.add( responses.GET, re.compile( r"""^https://[^/job].*/job/Job-1/batch/Batch-1/result /752x000000000F2$""", re.X), body='[{"Id": "001xx000003DHP0AAO", "AccountId": "ID-13",' '"Email": "*****@*****.**","FirstName": "Bob",' '"LastName": "x"},{"Id": "001xx000003DHP1AAO",' '"AccountId": "ID-24","Email": "*****@*****.**",' '"FirstName": "Alice","LastName": "y"}]', status=http.OK) data = 'SELECT Id,AccountId,Email,FirstName,LastName FROM Contact' session = requests.Session() client = Salesforce(session_id=tests.SESSION_ID, instance_url=tests.SERVER_URL, session=session) contact = client.bulk.Contact.query_all(data) self.assertEqual(self.expected_query, contact)
def guess_format(request): file_format = json.loads(request.POST.get('fileFormat', '{}')) file_type = file_format['file_type'] path = urllib_unquote(file_format["path"]) if sys.version_info[0] < 3 and (file_type == 'excel' or path[-3:] == 'xls' or path[-4:] == 'xlsx'): return JsonResponse({ 'status': -1, 'message': 'Python2 based Hue does not support Excel file importer' }) if file_format['inputFormat'] == 'localfile': if file_type == 'excel': format_ = {"type": "excel", "hasHeader": True} else: format_ = { "quoteChar": "\"", "recordSeparator": '\\n', "type": "csv", "hasHeader": True, "fieldSeparator": "," } elif file_format['inputFormat'] == 'file': if path[-3:] == 'xls' or path[-4:] == 'xlsx': file_obj = request.fs.open(path) if path[-3:] == 'xls': df = pd.read_excel(file_obj.read(1024 * 1024 * 1024), engine='xlrd') else: df = pd.read_excel(file_obj.read(1024 * 1024 * 1024), engine='openpyxl') _csv_data = df.to_csv(index=False) path = excel_to_csv_file_name_change(path) request.fs.create(path, overwrite=True, data=_csv_data) indexer = MorphlineIndexer(request.user, request.fs) if not request.fs.isfile(path): raise PopupException( _('Path %(path)s is not a file') % file_format) stream = request.fs.open(path) format_ = indexer.guess_format( {"file": { "stream": stream, "name": path }}) _convert_format(format_) if file_format["path"][-3:] == 'xls' or file_format["path"][ -4:] == 'xlsx': format_ = { "quoteChar": "\"", "recordSeparator": '\\n', "type": "excel", "hasHeader": True, "fieldSeparator": "," } elif file_format['inputFormat'] == 'table': db = dbms.get(request.user) try: table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName']) except Exception as e: raise PopupException( e.message if hasattr(e, 'message') and e.message else e) storage = {} for delim in table_metadata.storage_details: if delim['data_type']: if '=' in delim['data_type']: key, val = delim['data_type'].split('=', 1) storage[key] = val else: storage[delim['data_type']] = delim['comment'] if table_metadata.details['properties']['format'] == 'text': format_ = { "quoteChar": "\"", "recordSeparator": '\\n', "type": "csv", "hasHeader": False, "fieldSeparator": storage.get('field.delim', ',') } elif table_metadata.details['properties']['format'] == 'parquet': format_ = { "type": "parquet", "hasHeader": False, } else: raise PopupException( 'Hive table format %s is not supported.' % table_metadata.details['properties']['format']) elif file_format['inputFormat'] == 'query': format_ = { "quoteChar": "\"", "recordSeparator": "\\n", "type": "csv", "hasHeader": False, "fieldSeparator": "\u0001" } elif file_format['inputFormat'] == 'rdbms': format_ = {"type": "csv"} elif file_format['inputFormat'] == 'stream': if file_format['streamSelection'] == 'kafka': format_ = { "type": "json", # "fieldSeparator": ",", # "hasHeader": True, # "quoteChar": "\"", # "recordSeparator": "\\n", 'topics': get_topics(request.user) } elif file_format['streamSelection'] == 'flume': format_ = { "type": "csv", "fieldSeparator": ",", "hasHeader": True, "quoteChar": "\"", "recordSeparator": "\\n" } elif file_format['inputFormat'] == 'connector': if file_format['connectorSelection'] == 'sfdc': sf = Salesforce(username=file_format['streamUsername'], password=file_format['streamPassword'], security_token=file_format['streamToken']) format_ = { "type": "csv", "fieldSeparator": ",", "hasHeader": True, "quoteChar": "\"", "recordSeparator": "\\n", 'objects': [ sobject['name'] for sobject in sf.restful('sobjects/')['sobjects'] if sobject['queryable'] ] } else: raise PopupException( _('Input format %(inputFormat)s connector not recognized: $(connectorSelection)s' ) % file_format) else: raise PopupException( _('Input format not recognized: %(inputFormat)s') % file_format) format_['status'] = 0 return JsonResponse(format_)
def guess_field_types(request): file_format = json.loads(request.POST.get('fileFormat', '{}')) if file_format['inputFormat'] == 'file': indexer = MorphlineIndexer(request.user, request.fs) path = urllib_unquote(file_format["path"]) stream = request.fs.open(path) encoding = check_encoding(stream.read(10000)) stream.seek(0) _convert_format(file_format["format"], inverse=True) format_ = indexer.guess_field_types({ "file": { "stream": stream, "name": path }, "format": file_format['format'] }) # Note: Would also need to set charset to table (only supported in Hive) if 'sample' in format_ and format_['sample']: format_['sample'] = escape_rows(format_['sample'], nulls_only=True, encoding=encoding) for col in format_['columns']: col['name'] = smart_unicode(col['name'], errors='replace', encoding=encoding) elif file_format['inputFormat'] == 'table': sample = get_api(request, { 'type': 'hive' }).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName']) db = dbms.get(request.user) table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName']) format_ = { "sample": sample['rows'][:4], "columns": [ Field(col.name, HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type, 'string')).to_dict() for col in table_metadata.cols ] } elif file_format['inputFormat'] == 'query': query_id = file_format['query']['id'] if file_format['query'].get( 'id') else file_format['query'] notebook = Notebook(document=Document2.objects.document( user=request.user, doc_id=query_id)).get_data() snippet = notebook['snippets'][0] db = get_api(request, snippet) if file_format.get('sampleCols'): columns = file_format.get('sampleCols') sample = file_format.get('sample') else: snippet['query'] = snippet['statement'] try: sample = db.fetch_result(notebook, snippet, 4, start_over=True)['rows'][:4] except Exception as e: LOG.warning( 'Skipping sample data as query handle might be expired: %s' % e) sample = [[], [], [], [], []] columns = db.autocomplete(snippet=snippet, database='', table='') columns = [ Field( col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict() for col in columns['extended_columns'] ] format_ = { "sample": sample, "columns": columns, } elif file_format['inputFormat'] == 'rdbms': api = _get_api(request) sample = api.get_sample_data(None, database=file_format['rdbmsDatabaseName'], table=file_format['tableName']) format_ = { "sample": list(sample['rows'])[:4], "columns": [ Field(col['name'], col['type']).to_dict() for col in sample['full_headers'] ] } elif file_format['inputFormat'] == 'stream': if file_format['streamSelection'] == 'kafka': data = get_topic_data(request.user, file_format.get('kafkaSelectedTopics')) kafkaFieldNames = [col['name'] for col in data['full_headers']] kafkaFieldTypes = [col['type'] for col in data['full_headers']] topics_data = data['rows'] format_ = { "sample": topics_data, "columns": [ Field(col, 'string', unique=False).to_dict() for col in kafkaFieldNames ] } # data = """%(kafkaFieldNames)s # %(data)s""" % { # 'kafkaFieldNames': ','.join(kafkaFieldNames), # 'data': '\n'.join([','.join(cols) for cols in topics_data]) # } # stream = string_io() # stream.write(data) # _convert_format(file_format["format"], inverse=True) # indexer = MorphlineIndexer(request.user, request.fs) # format_ = indexer.guess_field_types({ # "file": { # "stream": stream, # "name": file_format['path'] # }, # "format": file_format['format'] # }) # type_mapping = dict( # list( # zip(kafkaFieldNames, kafkaFieldTypes) # ) # ) # for col in format_['columns']: # col['keyType'] = type_mapping[col['name']] # col['type'] = type_mapping[col['name']] elif file_format['streamSelection'] == 'flume': if 'hue-httpd/access_log' in file_format['channelSourcePath']: columns = [{ 'name': 'id', 'type': 'string', 'unique': True }, { 'name': 'client_ip', 'type': 'string' }, { 'name': 'time', 'type': 'date' }, { 'name': 'request', 'type': 'string' }, { 'name': 'code', 'type': 'plong' }, { 'name': 'bytes', 'type': 'plong' }, { 'name': 'method', 'type': 'string' }, { 'name': 'url', 'type': 'string' }, { 'name': 'protocol', 'type': 'string' }, { 'name': 'app', 'type': 'string' }, { 'name': 'subapp', 'type': 'string' }] else: columns = [{'name': 'message', 'type': 'string'}] format_ = { "sample": [['...'] * len(columns)] * 4, "columns": [ Field(col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get( col['type'], 'string'), unique=col.get('unique')).to_dict() for col in columns ] } elif file_format['inputFormat'] == 'connector': if file_format['connectorSelection'] == 'sfdc': sf = Salesforce(username=file_format['streamUsername'], password=file_format['streamPassword'], security_token=file_format['streamToken']) table_metadata = [{ 'name': column['name'], 'type': column['type'] } for column in sf.restful('sobjects/%(streamObject)s/describe/' % file_format)['fields']] query = 'SELECT %s FROM %s LIMIT 4' % (', '.join( [col['name'] for col in table_metadata]), file_format['streamObject']) print(query) try: records = sf.query_all(query) except SalesforceRefusedRequest as e: raise PopupException(message=str(e)) format_ = { "sample": [list(row.values())[1:] for row in records['records']], "columns": [ Field( col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get( col['type'], 'string')).to_dict() for col in table_metadata ] } else: raise PopupException( _('Connector format not recognized: %(connectorSelection)s') % file_format) else: raise PopupException( _('Input format not recognized: %(inputFormat)s') % file_format) return JsonResponse(format_)
def guess_format(request): file_format = json.loads(request.POST.get('fileFormat', '{}')) if file_format['inputFormat'] == 'file': path = urllib_unquote(file_format["path"]) indexer = MorphlineIndexer(request.user, request.fs) if not request.fs.isfile(path): raise PopupException( _('Path %(path)s is not a file') % file_format) stream = request.fs.open(path) format_ = indexer.guess_format( {"file": { "stream": stream, "name": path }}) _convert_format(format_) elif file_format['inputFormat'] == 'table': db = dbms.get(request.user) try: table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName']) except Exception as e: raise PopupException( e.message if hasattr(e, 'message') and e.message else e) storage = {} for delim in table_metadata.storage_details: if delim['data_type']: if '=' in delim['data_type']: key, val = delim['data_type'].split('=', 1) storage[key] = val else: storage[delim['data_type']] = delim['comment'] if table_metadata.details['properties']['format'] == 'text': format_ = { "quoteChar": "\"", "recordSeparator": '\\n', "type": "csv", "hasHeader": False, "fieldSeparator": storage.get('field.delim', ',') } elif table_metadata.details['properties']['format'] == 'parquet': format_ = { "type": "parquet", "hasHeader": False, } else: raise PopupException( 'Hive table format %s is not supported.' % table_metadata.details['properties']['format']) elif file_format['inputFormat'] == 'query': format_ = { "quoteChar": "\"", "recordSeparator": "\\n", "type": "csv", "hasHeader": False, "fieldSeparator": "\u0001" } elif file_format['inputFormat'] == 'rdbms': format_ = {"type": "csv"} elif file_format['inputFormat'] == 'stream': if file_format['streamSelection'] == 'kafka': format_ = { "type": "json", # "fieldSeparator": ",", # "hasHeader": True, # "quoteChar": "\"", # "recordSeparator": "\\n", 'topics': get_topics(request.user) } elif file_format['streamSelection'] == 'flume': format_ = { "type": "csv", "fieldSeparator": ",", "hasHeader": True, "quoteChar": "\"", "recordSeparator": "\\n" } elif file_format['inputFormat'] == 'connector': if file_format['connectorSelection'] == 'sfdc': sf = Salesforce(username=file_format['streamUsername'], password=file_format['streamPassword'], security_token=file_format['streamToken']) format_ = { "type": "csv", "fieldSeparator": ",", "hasHeader": True, "quoteChar": "\"", "recordSeparator": "\\n", 'objects': [ sobject['name'] for sobject in sf.restful('sobjects/')['sobjects'] if sobject['queryable'] ] } else: raise PopupException( _('Input format %(inputFormat)s connector not recognized: $(connectorSelection)s' ) % file_format) else: raise PopupException( _('Input format not recognized: %(inputFormat)s') % file_format) format_['status'] = 0 return JsonResponse(format_)
format_ = {"quoteChar": "\"", "recordSeparator": '\\n', "type": "csv", "hasHeader": False, "fieldSeparator": storage.get('field.delim', ',')} elif table_metadata.details['properties']['format'] == 'parquet': format_ = {"type": "parquet", "hasHeader": False,} else: raise PopupException('Hive table format %s is not supported.' % table_metadata.details['properties']['format']) elif file_format['inputFormat'] == 'query': format_ = {"quoteChar": "\"", "recordSeparator": "\\n", "type": "csv", "hasHeader": False, "fieldSeparator": "\u0001"} elif file_format['inputFormat'] == 'rdbms': format_ = RdbmsIndexer(request.user, file_format['rdbmsType']).guess_format() elif file_format['inputFormat'] == 'stream': if file_format['streamSelection'] == 'kafka': format_ = {"type": "csv", "fieldSeparator": ",", "hasHeader": True, "quoteChar": "\"", "recordSeparator": "\\n", 'topics': get_topics()} elif file_format['streamSelection'] == 'sfdc': sf = Salesforce( username=file_format['streamUsername'], password=file_format['streamPassword'], security_token=file_format['streamToken'] ) format_ = {"type": "csv", "fieldSeparator": ",", "hasHeader": True, "quoteChar": "\"", "recordSeparator": "\\n", 'objects': [sobject['name'] for sobject in sf.restful('sobjects/')['sobjects'] if sobject['queryable']]} format_['status'] = 0 return JsonResponse(format_) def guess_field_types(request): file_format = json.loads(request.POST.get('fileFormat', '{}')) if file_format['inputFormat'] == 'file': indexer = MorphlineIndexer(request.user, request.fs) path = urllib.unquote(file_format["path"]) stream = request.fs.open(path)
elif file_format['inputFormat'] == 'rdbms': format_ = RdbmsIndexer(request.user, file_format['rdbmsType']).guess_format() elif file_format['inputFormat'] == 'stream': if file_format['streamSelection'] == 'kafka': format_ = { "type": "csv", "fieldSeparator": ",", "hasHeader": True, "quoteChar": "\"", "recordSeparator": "\\n", 'topics': get_topics() } elif file_format['streamSelection'] == 'sfdc': sf = Salesforce(username=file_format['streamUsername'], password=file_format['streamPassword'], security_token=file_format['streamToken']) format_ = { "type": "csv", "fieldSeparator": ",", "hasHeader": True, "quoteChar": "\"", "recordSeparator": "\\n", 'objects': [ sobject['name'] for sobject in sf.restful('sobjects/')['sobjects']
def __init__(self, username, password, token): ''' Constructor ''' self.api = Salesforce(username=username, password=password, security_token=token) super(SalesforceExtractor, self).__init__(self._incoming_table_class)