def connect(self, service_url, username, password):
session = requests.Session()
session.verify = '/etc/ssl/certs'
if len(username) > 0:
session.auth = (username, password)
return pyodata.Client(service_url, session)
def test_invalid_odata_version():
"""Check handling of request for invalid OData version implementation"""
with pytest.raises(PyODataException) as e_info:
pyodata.Client(SERVICE_URL, requests, 'INVALID VERSION')
assert str(e_info.value).startswith('No implementation for selected odata version')
def test_create_client_for_local_metadata(metadata):
"""Check client creation for valid use case with local metadata"""
client = pyodata.Client(SERVICE_URL, requests, metadata=metadata)
assert isinstance(client, pyodata.v2.service.Service)
assert len(client.schema.entity_sets) != 0
def from_args(cls, args):
if args.requests_cache:
import requests_cache
requests_cache.install_cache(args.requests_cache)
client = pyodata.Client(args.url, requests.Session(), retain_null=True)
return Context(client, getattr(args, 'include', set()),
getattr(args, 'skip', set()), args.url)
def __init__(self, service, host, port, client, user, password, ssl,
verify):
"""Parameters:
- service: id of the odata service (e.g. UI5/ABAP_REPOSITORY_SRV)
- host: string host name or IP of
- port: string TCP/IP port for abap application server
- client: string SAP client
- user: string user name
- password: string user password
- ssl: boolean to switch between http and https
- verify: boolean to switch SSL validation on/off
"""
if ssl:
protocol = 'https'
if port is None:
port = '443'
else:
protocol = 'http'
if port is None:
port = '80'
self._base_url = f'{protocol}://{host}:{port}/sap/opu/odata/{service}'
self._query_args = f'sap-client={client}&saml2=disabled'
self._user = user
self._auth = HTTPBasicAuth(user, password)
self._timeout = config_get('http_timeout')
self._session = requests.Session()
self._session.verify = verify
self._session.auth = (user, password)
# csrf token handling for all future "create" requests
try:
get_logger().info(
'Executing head request as part of CSRF authentication %s',
self._base_url)
req = requests.Request('HEAD',
self._base_url,
headers={'x-csrf-token': 'fetch'})
req = self._session.prepare_request(req)
res = self._session.send(req, timeout=self._timeout)
except requests.exceptions.ConnectTimeout as ex:
raise TimedOutRequestError(req, self._timeout) from ex
if res.status_code == 401:
raise UnauthorizedError(req, res, self._user)
if res.status_code >= 400:
raise HTTPRequestError(req, res)
token = res.headers.get('x-csrf-token', '')
self._session.headers.update({'x-csrf-token': token})
# instance of the service
self.client = pyodata.Client(self._base_url, self._session)
def test_create_service_text_xml(metadata):
"""Check client creation for valid use case with MIME type 'text/xml'"""
responses.add(responses.GET,
"{0}/$metadata".format(SERVICE_URL),
content_type='text/xml',
body=metadata,
status=200)
client = pyodata.Client(SERVICE_URL, requests)
assert isinstance(client, pyodata.v2.service.Service)
# onw more test for '/' terminated url
client = pyodata.Client(SERVICE_URL + '/', requests)
assert isinstance(client, pyodata.v2.service.Service)
def test_create_service_application_xml(metadata):
"""Check client creation for valid use case with MIME type 'application/xml'"""
responses.add(
responses.GET,
f"{SERVICE_URL}/$metadata",
content_type='application/xml',
body=metadata,
status=200)
client = pyodata.Client(SERVICE_URL, requests)
assert isinstance(client, pyodata.v2.service.Service)
# onw more test for '/' terminated url
client = pyodata.Client(SERVICE_URL + '/', requests)
assert isinstance(client, pyodata.v2.service.Service)
assert client.schema.is_valid == True
def test_create_service_application(metadata, content_type):
"""Check client creation for valid MIME types"""
responses.add(
responses.GET,
f"{SERVICE_URL}/$metadata",
content_type=content_type,
body=metadata,
status=200)
client = pyodata.Client(SERVICE_URL, requests)
assert isinstance(client, pyodata.v2.service.Service)
# one more test for '/' terminated url
client = pyodata.Client(SERVICE_URL + '/', requests)
assert isinstance(client, pyodata.v2.service.Service)
assert client.schema.is_valid
def test_client_custom_configuration(mock_warning, metadata):
"""Check client creation for custom configuration"""
responses.add(
responses.GET,
f"{SERVICE_URL}/$metadata",
content_type='application/xml',
body=metadata,
status=200)
namespaces = {
'edmx': "customEdmxUrl.com",
'edm': 'customEdmUrl.com'
}
custom_config = Config(
xml_namespaces=namespaces,
default_error_policy=PolicyFatal(),
custom_error_policies={
ParserError.ANNOTATION: PolicyWarning(),
ParserError.ASSOCIATION: PolicyIgnore()
})
with pytest.raises(PyODataException) as e_info:
client = pyodata.Client(SERVICE_URL, requests, config=custom_config, namespaces=namespaces)
assert str(e_info.value) == 'You cannot pass namespaces and config at the same time'
client = pyodata.Client(SERVICE_URL, requests, namespaces=namespaces)
mock_warning.assert_called_with(
'Passing namespaces directly is deprecated. Use class Config instead',
DeprecationWarning
)
assert isinstance(client, pyodata.v2.service.Service)
assert client.schema.config.namespaces == namespaces
client = pyodata.Client(SERVICE_URL, requests, config=custom_config)
assert isinstance(client, pyodata.v2.service.Service)
assert client.schema.config == custom_config
def test_metadata_not_reachable():
"""Check handling of not reachable service metadata"""
responses.add(responses.GET,
"{0}/$metadata".format(SERVICE_URL),
content_type='text/html',
status=404)
with pytest.raises(HttpError) as e_info:
pyodata.Client(SERVICE_URL, requests)
assert str(e_info.value).startswith('Metadata request failed')
def test_metadata_saml_not_authorized():
"""Check handling of not SAML / OAuth unauthorized response"""
responses.add(responses.GET,
"{0}/$metadata".format(SERVICE_URL),
content_type='text/html; charset=utf-8',
status=200)
with pytest.raises(HttpError) as e_info:
pyodata.Client(SERVICE_URL, requests)
assert str(e_info.value).startswith(
'Metadata request did not return XML, MIME type:')
def authenticate(self):
resp = self.session.post('{}/api/Account/Login'.format(self.url),
json={
'UserId': self.userid,
'Password': self.password,
'AppKey': self.appkey
})
if resp.status_code == 200:
data = resp.json()
self.session.headers.update(
{'Authorization': 'Basic {}'.format(data['SecurityToken'])})
self.odata = pyodata.Client('{}/OData'.format(self.url),
self.session)
else:
raise ClientError('Unable to authenticate to Field Direct')
def client():
with open(metadata_fixture, 'rb') as metadata_file:
local_metadata = metadata_file.read()
return pyodata.Client(SERVICE_URL, requests.Session(), metadata=local_metadata)
def execute(self, context):
step_msg = None
result = None
try:
if self.http_conn_id:
if self.http_hook and isinstance(self.http_hook, HttpHook):
self.log.info(
"`http_conn_id` is ignored when `http_hook` is provided."
)
else:
self.log.info("`http_hook` is not provided or invalid. "
"Trying `http_conn_id` to create HttpHook.")
self.http_hook = HttpHook(http_conn_id=self.http_conn_id)
if not self.http_hook:
step_msg = "creating http session"
raise AirflowException(
"Cannot operate without `http_hook` or `http_conn_id`.")
else:
url = requests.urllib3.util.url.parse_url(self.service_url)
self.http_hook.base_url = f"{url.scheme}://{url.host}" + f"{':' + str(url.port) if url.port else ''}"
with self.http_hook.get_conn() as sess:
client = pyodata.Client(self.service_url, sess)
if self.function is not None and isinstance(
self.function, str):
if self.entity is not None:
self.log.info(
"entity is ignored when function is provided.")
step_msg = f"preparing data request for function {self.function}"
request = client.functions.__getattr__(self.function)
if self.parameters:
for name, value in self.parameters.items():
request = request.parameter(name, value)
step_msg = f"sending request and recieving data"
request = request.execute()
step_msg = f"creating result pandas DataFrame"
result = pd.DataFrame.from_dict(request['results'])
elif self.entity:
if not self.property:
self.log.info(
"`property` is not provided, filling with `entity` value"
)
self.property = self.entity
step_msg = f"preparing data request for entity set {self.entity}"
data = client.entity_sets.__getattr__(self.entity)\
.get_entity(**self.parameters).nav(self.property).get_entities()
step_msg = f"sending request and recieving data"
data = data.execute()
step_msg = f"recieving headers"
columns = {
es.name: es
for es in client.schema.entity_sets
}[self.property].entity_type.proprties()
step_msg = f"creating result pandas DataFrame"
result = pd.DataFrame.from_records([{
col.name: d.__getattr__(col.name)
for col in columns
} for d in data])
except Exception as e:
raise AirflowException(
f"Error while step \"{step_msg}\", error: {str(e)}")
return result
import pyodata
import requests
SERVICE_URL = 'http://services.odata.org/V2/Northwind/Northwind.svc/'
northwind = pyodata.Client(SERVICE_URL, requests.Session())
employees = northwind.entity_sets.Employees.get_entities().select(
'EmployeeID,LastName').execute()
for employee in employees:
print(employee.EmployeeID, employee.LastName)
print('test')
import requests
import re
import csv
exec(open("./CONFIDENTIAL_S41.py").read())
URL = 'http://' + host + '/sap/opu/odata/sap/ZCDS_DDDDLSRC_CDS'
#exec(open("./CONFIDENTIAL_SW1.py").read())
#URL = 'http://' + host + '/sap/opu/odata/SAAQ/BW_CDS_VIEW_CDS'
session = requests.Session()
session.auth = requests_auth
print(URL)
odata_service = pyodata.Client(URL, session)
print(odata_service)
print(odata_service.entity_sets.ZCDS_DDDDLSRC.get_entities().count().execute())
#print(odata_service.entity_sets.xSAAQxBW_CDS_VIEW.get_entities().count().execute())
#exit()
with open('DDDDLSRC.tsv', "w+", newline='') as output_csv:
writer = csv.writer(output_csv, delimiter='\t')
writer.writerow(['ddlname', 'number', 'source'])
ctr = 0
for row in odata_service.entity_sets.ZCDS_DDDDLSRC.get_entities().execute(
import pyodata
from pyodata.v2.model import PolicyFatal, PolicyWarning, ParserError, Config
custom_config = Config(default_error_policy=PolicyWarning(),
custom_error_policies={
ParserError.ANNOTATION: PolicyWarning(),
ParserError.ASSOCIATION: PolicyWarning()
})
FILE_NAME = 'metadata.xml'
with open(FILE_NAME, 'rb') as mtd_file:
local_metadata = mtd_file.read()
sample = pyodata.Client('NOT_VALID',
None,
metadata=local_metadata,
config=custom_config)
print('Metadata validation status: ', sample.schema.is_valid)
import pyodata
import requests
# https://github.com/SAP/python-pyodata/issues/83
SERVICE_URL = "http://knesset.gov.il/Odata/ParliamentInfo.svc/"
client = pyodata.Client(SERVICE_URL, requests.Session())
client.entity_sets.KNS_Position.get_entities().count().execute()
request = client.entity_sets.KNS_Position.get_entities()
for pos in request.execute():
print(pos)
import pyodata
from requests import Request, Session
from pyodata.v2.model import PolicyFatal, PolicyWarning, PolicyIgnore, ParserError, Config
SERVICE_URL = 'http://psl-e.one-erp.telekom.de/sap/opu/odata/SAP/ZPSL_GWSAMPLE_BASIC_SRV/'
session = Session()
# 400
session.auth = ('44544331', 'Fhm9Z2478p!EW')
# 200
# session.auth = ('44544331', 'xmXxPj6GXZHa!')
session.params = {'sap-client': 400, 'sap-language': 'EN'}
namespaces = {
'atom': 'http://www.w3.org/2005/Atom',
'app': 'http://www.w3.org/2007/app'
}
custom_config = Config(xml_namespaces=namespaces,
default_error_policy=PolicyFatal(),
custom_error_policies={ParserError.ANNOTATION: PolicyWarning(),
ParserError.ASSOCIATION: PolicyIgnore()})
services = pyodata.Client(url=SERVICE_URL, connection=session, config=custom_config)
bp_request = services.entity_sets.BusinessPartnerSet.get_entities()
for item in bp_request.select('BusinessPartnerID,CompanyName').execute():
# print(item.BusinessPartnerID, item.CompanyName)
print(item.EmailAddress)
import pyodata
# Set the PAT token up as an environment variable DevOpsPAT and it is used with
# os.environ["DevOpsPAT"]
# Set the User name up as an environment variable DevOpsUser and it is used with
# os.environ["DevOpsUser"]
# For now I have hard coded with my org
org_name="samuelrowe-ms"
project="Home%20Improvements"
version="v2.0"
session = requests.Session()
session.auth = (os.environ["DevOpsUser"],os.environ["DevOpsPAT"])
#session.get("https://analytics.dev.azure.com/samuelrowe-ms/Home%20Improvements/_odata/v2.0/WorkItems/?$Select=WorkItemId")
WORK_ITEM_URL = ( "https://analytics.dev.azure.com/" + org_name + "/" + project + "/_odata/" + version + "/WorkItems/?$Select=WorkItemId" )
r = session.get(WORK_ITEM_URL)
print(r.text)
SERVICE_URL = ( "https://analytics.dev.azure.com/" + org_name + "/" + project + "/_odata/" + version + "/" )
# This falls over ? I think its because it doesn't understand OData v4
devops = pyodata.Client(SERVICE_URL, session)
work_id_list = devops.entity_sets.WorkItems.get_entities()
print(work_id_list)
# odata feed connection
environment = 'SW1'
host = 'TO_BE_DEFINED'
exec(open('./CONFIDENTIAL_' + environment + '.py').read())
SERVICE_URL = 'http://' + host + '/sap/opu/odata/SAAQ/BW_RSOADSOT_CDS'
EntityName = 'xSAAQxBW_RSOADSOT'
print('OData service URL:', SERVICE_URL)
session = requests.Session()
session.auth = requests_auth
odata_feed = pyodata.Client(SERVICE_URL, session)
# number of entries
print(
'xSAAQxBW_RSOADSOT:',
odata_feed.entity_sets.xSAAQxBW_RSOADSOT.get_entities().count().execute())
#exit()
# list column names
rows = odata_feed.entity_sets.xSAAQxBW_RSOADSOT.get_entities().execute()
row = rows[0].__dict__['_cache']
print('colonnes: ', row)
adso = row['adsonm']
name = row['colname'].replace('!23!2F!2F!2F', '').replace('!2F', '/')
import requests
import pyodata
metadata_ = requests.get('https://api.parliament.uk/odata/$metadata',
headers={
'Accept': 'application/xml'
}).text.encode()
client = pyodata.Client('https://api.parliament.uk/odata',
requests.Session(),
metadata=metadata_)
def __init__(self, url, session=None):
self.session = session if session is not None else requests.session()
self._client = pyodata.Client(url, self.session)
