def __init__(self,
hostname: Optional[str] = None,
root: Optional[str] = None,
**kwargs) -> None:
super().__init__()
# --- internal ---
self._file_system = None # type: Optional[str]
self._http_authorization = 'Authorization'
self._http_client = CdmHttpClient() # type: CdmHttpClient
self._http_xms_date = 'x-ms-date'
self._http_xms_version = 'x-ms-version'
self._resource = "https://storage.azure.com" # type: Optional[str]
self._type = 'adls'
self._root = None
self._sub_path = None # type: Optional[str]
if root and hostname:
self.root = root # type: Optional[str]
self.hostname = hostname # type: Optional[str]
self.tenant = kwargs.get('tenant', None) # type: Optional[str]
self.client_id = kwargs.get('client_id',
None) # type: Optional[str]
self.secret = kwargs.get('secret', None) # type: Optional[str]
self.shared_key = kwargs.get('shared_key',
None) # type: Optional[str]
self.location_hint = None # type: Optional[str]
# --- internal ---
self._auth_context = adal.AuthenticationContext(
'https://login.windows.net/' +
self.tenant) if self.tenant else Non
def __init__(self, endpoint: Optional[str] = None, **kwargs) -> None:
super().__init__()
super(NetworkAdapter, self).__init__()
super(StorageAdapterBase, self).__init__()
# --- internal ---
self._formatted_endpoint = None # type: Optional[str]
self._http_authorization = 'Authorization'
self._http_client = CdmHttpClient() # type: CdmHttpClient
self._http_xms_continuation = 'x-ms-continuation'
self._http_xms_date = 'x-ms-date'
self._http_xms_version = 'x-ms-version'
self._scope = ['https://dev.azuresynapse.net/.default'] # type: Optional[List[str]]
self._type = 'Syms'
self.http_max_results = self.HTTP_DEFAULT_MAX_RESULTS # type: int
self.timeout = self.SYMS_DEFAULT_TIMEOUT # type: int
if endpoint:
self._endpoint = self._format_endpoint(endpoint) # type: Optional[str]
self._base_uri = 'https://{}/databases'.format(self._endpoint)
self.client_id = kwargs.get('client_id', None) # type: Optional[str]
self.secret = kwargs.get('secret', None) # type: Optional[str]
self.token_provider = kwargs.get('token_provider', None) # type: Optional[TokenProvider]
self.azure_endpoint = kwargs.get('azure_endpoint', AzureCloudEndpoint.AZURE_PUBLIC) # type: AzureCloudEndpoint
# --- internal ---
self._tenant = kwargs.get('tenant', None) # type: Optional[str]
self._auth_context = None
def __init__(self, ctx: 'CdmCorpusContext',
telemetry_config: 'TelemetryConfig'):
# Maximum number of retries allowed for an HTTP request
self.max_num_retries = 5
# Maximum timeout for completing an HTTP request including retries.
self.max_timeout_milliseconds = 10000
# Maximum timeout for a single HTTP request.
self.timeout_milliseconds = 1000
# --- internal ---
# The CDM corpus context.
self._ctx = ctx # type: CdmCorpusContext
# The Kusto configuration.
self._config = telemetry_config # type: TelemetryConfig
# Queuing all log ingestion request to Kusto.
self._request_queue = Queue(
) # type: Queue[tuple[CdmStatusLevel, str]]
# An HTTP client for post requests which ingests data into Kusto.
self._http_client = CdmHttpClient() # type: CdmHttpClient
def __init__(self, hostname: Optional[str] = None, root: Optional[str] = None, **kwargs) -> None:
super().__init__()
super(NetworkAdapter, self).__init__()
super(StorageAdapterBase, self).__init__()
# --- internal ---
self._adapter_paths = {} # type: Dict[str, str]
self._root_blob_contrainer = None # type: Optional[str]
self._formatted_hostname = None # type: Optional[str]
self._http_authorization = 'Authorization'
self._http_client = CdmHttpClient() # type: CdmHttpClient
self._http_xms_continuation = 'x-ms-continuation'
self._http_xms_date = 'x-ms-date'
self._http_xms_version = 'x-ms-version'
self._http_xms_version = 'x-ms-version'
self._resource = "https://storage.azure.com" # type: Optional[str]
self._type = 'adls'
self._root = None
self._unescaped_root_sub_path = None # type: Optional[str]
self._escaped_root_sub_path = None # type: Optional[str]
self._file_modified_time_cache = {} # type: Dict[str, datetime]
self.http_max_results = self.HTTP_DEFAULT_MAX_RESULTS # type: int
self.timeout = self.ADLS_DEFAULT_TIMEOUT # type: int
if root and hostname:
self.root = root # type: Optional[str]
self.hostname = hostname # type: Optional[str]
self.client_id = kwargs.get('client_id', None) # type: Optional[str]
self.secret = kwargs.get('secret', None) # type: Optional[str]
self.shared_key = kwargs.get('shared_key', None) # type: Optional[str]
self.token_provider = kwargs.get('token_provider', None) # type: Optional[TokenProvider]
# --- internal ---
self._tenant = kwargs.get('tenant', None) # type: Optional[str]
self._auth_context = adal.AuthenticationContext('https://login.windows.net/' + self.tenant) if self.tenant else None
def __init__(self, hosts: Dict[str, str], **kwargs) -> None:
super().__init__(kwargs.get('http_config', {}))
self._sources = {} # type: Dict[str, str]
self._sources_by_id = {} # type: Dict[str, Dict[str, str]]
for key, value in hosts.items():
self._fetch_or_register_host_info(value, key)
self.http_client = CdmHttpClient() # type: CdmHttpClient
def __init__(self) -> None:
super().__init__()
self.location_hint = None
# --- internal ---
self._type = 'github'
self._is_folder = {} # type: Dict[str, bool]
self._http_client = CdmHttpClient(self._raw_root) # type: CdmHttpClient
def __init__(self) -> None:
super().__init__()
warnings.warn('GithubAdapter is deprecated. Please use the CdmStandardsAdapter instead.', DeprecationWarning)
self.location_hint = None
# --- internal ---
self._type = 'github'
self._is_folder = {} # type: Dict[str, bool]
self._http_client = CdmHttpClient(self._raw_root) # type: CdmHttpClient
def __init__(self, root: str, hostname: str, resource: str, client_id: str,
secret: str, tenant: str, **kwargs) -> None:
super().__init__(kwargs.get('http_config', {}))
self._root = root # type: str
self._hostname = hostname # type: str
self._resource = resource # type: str
self._client_id = client_id # type: str
self._client_secret = secret # type: str
self._auth_context = adal.AuthenticationContext(
'https://login.windows.net/' + tenant)
self.http_client = CdmHttpClient() # type: CdmHttpClient
def __init__(self) -> None:
super().__init__()
super(NetworkAdapter, self).__init__()
super(StorageAdapterBase, self).__init__()
warnings.warn(
'GithubAdapter is deprecated. Please use the CdmStandardsAdapter instead.',
DeprecationWarning)
# --- internal ---
self._type = 'github'
self._http_client = CdmHttpClient(
self._raw_root) # type: CdmHttpClient
def __init__(self, root: str = '/logical') -> None:
""""Constructs a CdmStandardsAdapter.
# Parameters:
# root: The root path specifies either to read the standard files in logical or resolved form.
"""
super().__init__()
self.location_hint = None
self.root = root
# --- internal ---
self._type = 'cdm-standards'
self._http_client = CdmHttpClient(
self._STANDARDS_ENDPOINT) # type: CdmHttpClient
async def test_result_returned_successfully(self):
"""Testing for a successful return of a result from Cdm Http Client."""
github_url = 'https://raw.githubusercontent.com/'
corpus_path = '/Microsoft/CDM/master/schemaDocuments/foundations.cdm.json'
cdm_http_client = CdmHttpClient(github_url)
cdm_http_request = CdmHttpRequest(corpus_path, 1, 'GET')
cdm_http_request.timeout = 5000
cdm_http_request.maximum_timeout = 10000
cdm_http_request.headers = {'User-Agent': 'CDM'}
cdm_http_response = await cdm_http_client._send_async(cdm_http_request, self.call_back)
self.assertEqual(200, cdm_http_response.status_code)
def __init__(self, hosts: Optional[Dict[str, str]] = None) -> None:
super().__init__()
super(NetworkAdapter, self).__init__()
super(StorageAdapterBase, self).__init__()
# --- internal ---
self._hosts = {} # type: Dict[str, str]
self._sources = {} # type: Dict[str, str]
self._sources_by_id = {} # type: Dict[str, Dict[str, str]]
self._type = 'remote'
self._http_client = CdmHttpClient() # type: CdmHttpClient
if hosts:
self.hosts = hosts
def __init__(self, hosts: Optional[Dict[str, str]] = None) -> None:
super().__init__()
self.location_hint = None # type: Optional[str]
# --- internal ---
self._hosts = {} # type: Dict[str, str]
self._sources = {} # type: Dict[str, str]
self._sources_by_id = {} # type: Dict[str, Dict[str, str]]
self._type = 'remote'
self._http_client = CdmHttpClient() # type: CdmHttpClient
if hosts:
self.hosts = hosts
async def test_number_of_retries_exceeded_exception(self):
"""Testing for a failed return of a result back from Cdm Http Client."""
github_url = 'https://raw.githubusercontent2.com/'
corpus_path = '/Microsoft/CDM/master/schemaDocuments/foundations.cdm.json'
cdm_http_client = CdmHttpClient(github_url)
cdm_http_request = CdmHttpRequest(corpus_path, 1, 'GET')
cdm_http_request.timeout = 5000
cdm_http_request.maximum_timeout = 10000
cdm_http_request.headers = {'User-Agent': 'CDM'}
try:
await cdm_http_client._send_async(cdm_http_request, self.call_back)
except Exception as e:
self.assertRaises(CdmNumberOfRetriesExceededException)
def __init__(self,
hostname: Optional[str] = None,
root: Optional[str] = None,
**kwargs) -> None:
super().__init__()
super(NetworkAdapter, self).__init__()
super(StorageAdapterBase, self).__init__()
# --- internal ---
self._adapter_paths = {} # type: Dict[str, str]
self._root_blob_container = None # type: Optional[str]
self._http_authorization = 'Authorization'
self._http_client = CdmHttpClient() # type: CdmHttpClient
self._http_xms_continuation = 'x-ms-continuation'
self._http_xms_date = 'x-ms-date'
self._http_xms_version = 'x-ms-version'
self._http_xms_version = 'x-ms-version'
self._scope = ['https://storage.azure.com/.default'
] # type: Optional[List[str]]
self._type = 'adls'
self._root = None
self._sas_token = None
self._unescaped_root_sub_path = None # type: Optional[str]
self._escaped_root_sub_path = None # type: Optional[str]
self._file_modified_time_cache = {} # type: Dict[str, datetime]
self.http_max_results = self.HTTP_DEFAULT_MAX_RESULTS # type: int
self.timeout = self.ADLS_DEFAULT_TIMEOUT # type: int
if root and hostname:
self.root = root # type: Optional[str]
self.hostname = hostname # type: Optional[str]
self.client_id = kwargs.get('client_id',
None) # type: Optional[str]
self.secret = kwargs.get('secret', None) # type: Optional[str]
self.shared_key = kwargs.get('shared_key',
None) # type: Optional[str]
self.sas_token = kwargs.get('sas_token',
None) # type: Optional[str]
self.token_provider = kwargs.get(
'token_provider', None) # type: Optional[TokenProvider]
self.endpoint = kwargs.get(
'endpoint',
AzureCloudEndpoint.AZURE_PUBLIC) # type: AzureCloudEndpoint
# --- internal ---
self._tenant = kwargs.get('tenant', None) # type: Optional[str]
self._auth_context = None
def __init__(self, **kwargs) -> None:
super().__init__(kwargs.get('http_config', {}))
self._is_folder = {} # type: Dict[str, bool]
self.http_client = CdmHttpClient(self._raw_root) # type: CdmHttpClient