def submit_adla_job(client,
account_name,
job_name,
script,
runtime_version=None,
compile_mode=None,
compile_only=False,
degree_of_parallelism=1,
priority=1000):
if not script or len(script) < 1:
# pylint: disable=line-too-long
raise CLIError('Could not read script content from the supplied --script param. It is either empty or an invalid file')
job_properties = USqlJobProperties(script)
if runtime_version:
job_properties.runtime_version = runtime_version
if compile_mode:
job_properties.compile_mode = compile_mode
submit_params = JobInformation(job_name,
JobType.usql,
job_properties,
degree_of_parallelism,
priority)
if compile_only:
return client.build(account_name, submit_params)
job_id = _get_uuid_str()
return client.create(account_name, job_id, submit_params)
def submit_adla_job(client,
account_name,
job_name,
script,
runtime_version=None,
compile_mode=None,
compile_only=False,
degree_of_parallelism=1,
priority=1000):
if not script or len(script) < 1:
# pylint: disable=line-too-long
raise CLIError(
'Could not read script content from the supplied --script param. It is either empty or an invalid file'
)
job_properties = USqlJobProperties(script)
if runtime_version:
job_properties.runtime_version = runtime_version
if compile_mode:
job_properties.compile_mode = compile_mode
submit_params = JobInformation(job_name, JobType.usql, job_properties,
degree_of_parallelism, priority)
if compile_only:
return client.build(account_name, submit_params)
job_id = _get_uuid_str()
return client.create(account_name, job_id, submit_params)
def add_usql_job(scripts_folder, directory_name, usql_file, adl_token,
adl_name, simulation_datetime, au_per_usql_job):
#may need to recreate adl_token every time in case it expires
#adl_token = lib.auth(tenant_id=adl_tenant_id, client_id=adl_client_id, client_secret=adl_client_secret)
adla_job_client = DataLakeAnalyticsJobManagementClient(
adl_token, 'azuredatalakeanalytics.net')
# download USQL file from ADLS
usql_file_full_path = scripts_folder + '/' + directory_name + '/' + usql_file + '.usql'
adls_file_system_client = core.AzureDLFileSystem(adl_token,
store_name=adl_name)
multithread.ADLDownloader(adls_file_system_client,
lpath='.',
rpath=usql_file_full_path,
overwrite=True)
usql_script = ''.join(open(usql_file + '.usql', 'r').readlines())
if simulation_datetime:
datetime_replace = "Convert.ToDateTime(\"" + simulation_datetime + "\")"
usql_script = usql_script.replace('DateTime.Now', datetime_replace)
jobId = str(uuid.uuid4())
jobInfo = JobInformation(name=directory_name + '/' + usql_file,
type='USql',
degree_of_parallelism=au_per_usql_job,
properties=USqlJobProperties(script=usql_script))
jobResult = adla_job_client.job.create(adl_name, jobId, jobInfo)
return (jobId)
def submit_job(self, account, job_submission):
if not self.__token_service.logged_in_user:
raise UserNotLoggedInError()
job_id = str(uuid4())
job_client = DataLakeAnalyticsJobManagementClient(self.__token_service.credentials, AdlaServiceSdkImpl.__adla_job_dns_suffix)
job_props = USqlJobProperties(script = job_submission.script, runtime_version = job_submission.runtime)
job_info = JobInformation(name = job_submission.name, type = AdlaServiceSdkImpl.__adla_job_type, properties = job_props, degree_of_parallelism = job_submission.parallelism, priority = job_submission.priority)
job = job_client.job.create(account, job_id, job_info)
return AdlaJob(job.job_id, job.name, job.type.name, job.submitter, job.degree_of_parallelism, job.priority, job.submit_time, job.start_time, job.end_time, job.state.name, job.result.name, account)
adla_account = 'tkdemo'
resource_group = 'rgDataLake'
sub_id = 'ae7513ac-36d5-4635-b466-e2230009cc8b'
job_client = DataLakeAnalyticsJobManagementClient(
creds, 'azuredatalakeanalytics.net')
script = '@a = SELECT * FROM (VALUES ("Hello, World!")) AS T(message); OUTPUT @a TO "/Samples/Output/HelloWorld.csv" USING Outputters.Csv();'
job_id = str(uuid.uuid4())
job_client.job.create(
adla_account, job_id,
JobInformation(name='HelloWorld',
type='USql',
properties=USqlJobProperties(script=script)))
job_result = job_client.job.get(adla_account, job_id)
while job_result.state != JobState.ended:
print('Job is not yet done. Waiting for 3 seconds. Current state: ' +
job_result.state.value)
time.sleep(3)
job_result = job_client.job.get(adla_account, job_id)
print('Job finished with result: ' + job_result.result.value)
def authenticate_username_password():
"""
Authenticate using user w/ username + password.