def submit_adla_job(client, account_name, job_name, script, runtime_version=None, compile_mode=None, compile_only=False, degree_of_parallelism=1, priority=1000): if not script or len(script) < 1: # pylint: disable=line-too-long raise CLIError('Could not read script content from the supplied --script param. It is either empty or an invalid file') job_properties = USqlJobProperties(script) if runtime_version: job_properties.runtime_version = runtime_version if compile_mode: job_properties.compile_mode = compile_mode submit_params = JobInformation(job_name, JobType.usql, job_properties, degree_of_parallelism, priority) if compile_only: return client.build(account_name, submit_params) job_id = _get_uuid_str() return client.create(account_name, job_id, submit_params)
def submit_adla_job(client, account_name, job_name, script, runtime_version=None, compile_mode=None, compile_only=False, degree_of_parallelism=1, priority=1000): if not script or len(script) < 1: # pylint: disable=line-too-long raise CLIError( 'Could not read script content from the supplied --script param. It is either empty or an invalid file' ) job_properties = USqlJobProperties(script) if runtime_version: job_properties.runtime_version = runtime_version if compile_mode: job_properties.compile_mode = compile_mode submit_params = JobInformation(job_name, JobType.usql, job_properties, degree_of_parallelism, priority) if compile_only: return client.build(account_name, submit_params) job_id = _get_uuid_str() return client.create(account_name, job_id, submit_params)
def add_usql_job(scripts_folder, directory_name, usql_file, adl_token, adl_name, simulation_datetime, au_per_usql_job): #may need to recreate adl_token every time in case it expires #adl_token = lib.auth(tenant_id=adl_tenant_id, client_id=adl_client_id, client_secret=adl_client_secret) adla_job_client = DataLakeAnalyticsJobManagementClient( adl_token, 'azuredatalakeanalytics.net') # download USQL file from ADLS usql_file_full_path = scripts_folder + '/' + directory_name + '/' + usql_file + '.usql' adls_file_system_client = core.AzureDLFileSystem(adl_token, store_name=adl_name) multithread.ADLDownloader(adls_file_system_client, lpath='.', rpath=usql_file_full_path, overwrite=True) usql_script = ''.join(open(usql_file + '.usql', 'r').readlines()) if simulation_datetime: datetime_replace = "Convert.ToDateTime(\"" + simulation_datetime + "\")" usql_script = usql_script.replace('DateTime.Now', datetime_replace) jobId = str(uuid.uuid4()) jobInfo = JobInformation(name=directory_name + '/' + usql_file, type='USql', degree_of_parallelism=au_per_usql_job, properties=USqlJobProperties(script=usql_script)) jobResult = adla_job_client.job.create(adl_name, jobId, jobInfo) return (jobId)
def submit_job(self, account, job_submission): if not self.__token_service.logged_in_user: raise UserNotLoggedInError() job_id = str(uuid4()) job_client = DataLakeAnalyticsJobManagementClient(self.__token_service.credentials, AdlaServiceSdkImpl.__adla_job_dns_suffix) job_props = USqlJobProperties(script = job_submission.script, runtime_version = job_submission.runtime) job_info = JobInformation(name = job_submission.name, type = AdlaServiceSdkImpl.__adla_job_type, properties = job_props, degree_of_parallelism = job_submission.parallelism, priority = job_submission.priority) job = job_client.job.create(account, job_id, job_info) return AdlaJob(job.job_id, job.name, job.type.name, job.submitter, job.degree_of_parallelism, job.priority, job.submit_time, job.start_time, job.end_time, job.state.name, job.result.name, account)
adla_account = 'tkdemo' resource_group = 'rgDataLake' sub_id = 'ae7513ac-36d5-4635-b466-e2230009cc8b' job_client = DataLakeAnalyticsJobManagementClient( creds, 'azuredatalakeanalytics.net') script = '@a = SELECT * FROM (VALUES ("Hello, World!")) AS T(message); OUTPUT @a TO "/Samples/Output/HelloWorld.csv" USING Outputters.Csv();' job_id = str(uuid.uuid4()) job_client.job.create( adla_account, job_id, JobInformation(name='HelloWorld', type='USql', properties=USqlJobProperties(script=script))) job_result = job_client.job.get(adla_account, job_id) while job_result.state != JobState.ended: print('Job is not yet done. Waiting for 3 seconds. Current state: ' + job_result.state.value) time.sleep(3) job_result = job_client.job.get(adla_account, job_id) print('Job finished with result: ' + job_result.result.value) def authenticate_username_password(): """ Authenticate using user w/ username + password.