def _submit_bundle(request, bundle, properties):
try:
deployment_mapping = {}
coords = dict([(c.uuid, c) for c in Document2.objects.filter(type='oozie-coordinator2', uuid__in=[b['coordinator'] for b in bundle.data['coordinators']])])
for i, bundled in enumerate(bundle.data['coordinators']):
coord = coords[bundled['coordinator']]
workflow = Workflow(document=coord.dependencies.filter(type='oozie-workflow2')[0])
wf_dir = Submission(request.user, workflow, request.fs, request.jt, properties).deploy()
deployment_mapping['wf_%s_dir' % i] = request.fs.get_hdfs_path(wf_dir)
coordinator = Coordinator(document=coord)
coord_dir = Submission(request.user, coordinator, request.fs, request.jt, properties).deploy()
deployment_mapping['coord_%s_dir' % i] = request.fs.get_hdfs_path(coord_dir)
deployment_mapping['coord_%s' % i] = coord
# Convert start/end dates of coordinator to server timezone
for prop in bundled['properties']:
if prop['name'] in ('end_date', 'start_date'):
prop['value'] = convert_to_server_timezone(prop['value'], local_tz=coordinator.data['properties']['timezone'])
properties.update(deployment_mapping)
submission = Submission(request.user, bundle, request.fs, request.jt, properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
LOG.exception('Error submitting bundle')
raise PopupException(_("Error submitting bundle %s") % (bundle,), detail=ex._headers.get('oozie-error-message', ex))
def _submit_bundle(request, bundle, properties):
try:
deployment_mapping = {}
coords = dict([(c.uuid, c) for c in Document2.objects.filter(type='oozie-coordinator2', uuid__in=[b['coordinator'] for b in bundle.data['coordinators']])])
for i, bundled in enumerate(bundle.data['coordinators']):
coord = coords[bundled['coordinator']]
workflow = Workflow(document=coord.dependencies.filter(type='oozie-workflow2')[0])
wf_dir = Submission(request.user, workflow, request.fs, request.jt, properties).deploy()
deployment_mapping['wf_%s_dir' % i] = request.fs.get_hdfs_path(wf_dir)
coordinator = Coordinator(document=coord)
coord_dir = Submission(request.user, coordinator, request.fs, request.jt, properties).deploy()
deployment_mapping['coord_%s_dir' % i] = request.fs.get_hdfs_path(coord_dir)
deployment_mapping['coord_%s' % i] = coord
# Convert start/end dates of coordinator to server timezone
for prop in bundled['properties']:
if prop['name'] in ('end_date', 'start_date'):
prop['value'] = convert_to_server_timezone(prop['value'], local_tz=coordinator.data['properties']['timezone'])
properties.update(deployment_mapping)
submission = Submission(request.user, bundle, request.fs, request.jt, properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
LOG.exception('Error submitting bundle')
raise PopupException(_("Error submitting bundle %s") % (bundle,), detail=ex._headers.get('oozie-error-message', ex), error_code=200)
def submit_external_job(request, application_path):
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
application_name = os.path.basename(application_path)
application_class = Bundle if application_name == 'bundle.xml' else Coordinator if application_name == 'coordinator.xml' else get_workflow()
mapping[application_class.get_application_path_key()] = os.path.dirname(application_path)
try:
submission = Submission(request.user, fs=request.fs, jt=request.jt, properties=mapping)
job_id = submission.run(application_path)
except RestException, ex:
detail = ex._headers.get('oozie-error-message', ex)
if 'Max retries exceeded with url' in str(detail):
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
LOG.exception(smart_str(detail))
raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail)
jsonify = request.POST.get('format') == 'json'
if jsonify:
return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'external_workflow'}, safe=False)
else:
request.info(_('Oozie job submitted'))
view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow'
return redirect(reverse('oozie:%s' % view, kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' % params_form.errors))
def _submit_coordinator(request, coordinator, mapping):
try:
wf = coordinator.workflow
wf_dir = Submission(
request.user,
wf,
request.fs,
request.jt,
mapping,
local_tz=coordinator.data['properties']['timezone']).deploy()
properties = {'wf_application_path': request.fs.get_hdfs_path(wf_dir)}
properties.update(mapping)
submission = Submission(request.user,
coordinator,
request.fs,
request.jt,
properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
LOG.exception('Error submitting coordinator')
raise PopupException(_("Error submitting coordinator %s") %
(coordinator, ),
detail=ex._headers.get('oozie-error-message', ex))
def _submit_bundle(request, bundle, properties):
try:
deployment_mapping = {}
coords = dict([(c.uuid, c) for c in Document2.objects.filter(type='oozie-coordinator2', uuid__in=[b['coordinator'] for b in bundle.data['coordinators']])])
for i, bundled in enumerate(bundle.data['coordinators']):
coord = coords[bundled['coordinator']]
workflow = Workflow(document=coord.dependencies.all()[0])
wf_dir = Submission(request.user, workflow, request.fs, request.jt, properties).deploy()
deployment_mapping['wf_%s_dir' % i] = request.fs.get_hdfs_path(wf_dir)
coordinator = Coordinator(document=coord)
coord_dir = Submission(request.user, coordinator, request.fs, request.jt, properties).deploy()
deployment_mapping['coord_%s_dir' % i] = coord_dir
deployment_mapping['coord_%s' % i] = coord
properties.update(deployment_mapping)
submission = Submission(request.user, bundle, request.fs, request.jt, properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
LOG.exception('Error submitting bundle')
raise PopupException(_("Error submitting bundle %s") % (bundle,), detail=ex._headers.get('oozie-error-message', ex))
def submit_external_job(request, application_path):
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
application_name = os.path.basename(application_path)
application_class = Bundle if application_name == 'bundle.xml' else Coordinator if application_name == 'coordinator.xml' else get_workflow()
mapping[application_class.get_application_path_key()] = application_path
try:
submission = Submission(request.user, fs=request.fs, jt=request.jt, properties=mapping)
job_id = submission.run(application_path)
except RestException, ex:
detail = ex._headers.get('oozie-error-message', ex)
if 'Max retries exceeded with url' in str(detail):
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
LOG.exception(smart_str(detail))
raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail)
request.info(_('Oozie job submitted'))
view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow'
return redirect(reverse('oozie:%s' % view, kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' % params_form.errors))
def _submit_bundle(request, bundle, properties):
try:
deployment_mapping = {}
coords = dict([(c.uuid, c) for c in Document2.objects.filter(
type='oozie-coordinator2',
uuid__in=[b['coordinator'] for b in bundle.data['coordinators']])])
for i, bundled in enumerate(bundle.data['coordinators']):
coord = coords[bundled['coordinator']]
workflow = Workflow(document=coord.dependencies.all()[0])
wf_dir = Submission(request.user, workflow, request.fs, request.jt,
properties).deploy()
deployment_mapping['wf_%s_dir' %
i] = request.fs.get_hdfs_path(wf_dir)
coordinator = Coordinator(document=coord)
coord_dir = Submission(request.user, coordinator, request.fs,
request.jt, properties).deploy()
deployment_mapping['coord_%s_dir' % i] = coord_dir
deployment_mapping['coord_%s' % i] = coord
properties.update(deployment_mapping)
submission = Submission(request.user,
bundle,
request.fs,
request.jt,
properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
raise PopupException(_("Error submitting bundle %s") % (bundle, ),
detail=ex._headers.get('oozie-error-message', ex))
def submit_external_job(request, application_path):
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if application_path.startswith('abfs:/') and not application_path.startswith('abfs://'):
application_path = application_path.replace("abfs:/", "abfs://")
elif application_path.startswith('s3a:/') and not application_path.startswith('s3a://'):
application_path = application_path.replace('s3a:/', 's3a://')
else:
application_path = "/" + application_path
if application_path.startswith("abfs://"):
application_path = abfspath(application_path)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
application_name = os.path.basename(application_path)
application_class = Bundle if application_name == 'bundle.xml' else Coordinator if application_name == 'coordinator.xml' else get_workflow()
mapping[application_class.get_application_path_key()] = os.path.dirname(application_path)
try:
submission = Submission(request.user, fs=request.fs, jt=request.jt, properties=mapping)
job_id = submission.run(application_path)
except RestException as ex:
detail = ex._headers.get('oozie-error-message', ex)
if 'Max retries exceeded with url' in str(detail):
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
LOG.exception(smart_str(detail))
raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail)
jsonify = request.POST.get('format') == 'json'
if jsonify:
return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'external_workflow'}, safe=False)
else:
request.info(_('Oozie job submitted'))
view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow'
return redirect(reverse('oozie:%s' % view, kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' % params_form.errors))
else:
parameters = Submission(request.user, fs=request.fs, jt=request.jt).get_external_parameters(application_path)
initial_params = ParameterForm.get_initial_params(parameters)
params_form = ParametersFormSet(initial=initial_params)
popup = render('editor/submit_job_popup.mako', request, {
'params_form': params_form,
'name': _('Job'),
'action': reverse('oozie:submit_external_job', kwargs={'application_path': application_path}),
'show_dryrun': os.path.basename(application_path) != 'bundle.xml',
'return_json': request.GET.get('format') == 'json'
}, force_template=True).content
return JsonResponse(popup, safe=False)
def _submit_workflow(user, fs, jt, workflow, mapping):
try:
submission = Submission(user, workflow, fs, jt, mapping)
job_id = submission.run()
return job_id
except RestException, ex:
detail = ex._headers.get('oozie-error-message', ex)
if 'Max retries exceeded with url' in str(detail):
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
LOG.error(smart_str(detail))
raise PopupException(_("Error submitting workflow %s") % (workflow,), detail=detail)
def _submit_workflow(user, fs, jt, workflow, mapping):
try:
submission = Submission(user, workflow, fs, jt, mapping)
job_id = submission.run()
workflow.document.add_to_history(submission.user, {'properties': submission.properties, 'oozie_id': submission.oozie_id})
return job_id
except RestException, ex:
detail = ex._headers.get('oozie-error-message', ex)
if 'Max retries exceeded with url' in str(detail):
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
LOG.exception('Error submitting workflow: %s' % smart_str(detail))
raise PopupException(_("Error submitting workflow %s: %s") % (workflow, detail))
def _submit_workflow(user, fs, jt, workflow, mapping):
try:
submission = Submission(user, workflow, fs, jt, mapping)
job_id = submission.run()
workflow.document.add_to_history(submission.user, {'properties': submission.properties, 'oozie_id': submission.oozie_id})
return job_id
except RestException, ex:
detail = ex._headers.get('oozie-error-message', ex)
if 'Max retries exceeded with url' in str(detail):
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
LOG.exception('Error submitting workflow: %s' % smart_str(detail))
raise PopupException(_("Error submitting workflow %s: %s") % (workflow, detail))
def _submit_coordinator(request, coordinator, mapping):
try:
wf_doc = Document2.objects.get(uuid=coordinator.data['properties']['workflow'])
wf_dir = Submission(request.user, Workflow(document=wf_doc), request.fs, request.jt, mapping).deploy()
properties = {'wf_application_path': request.fs.get_hdfs_path(wf_dir)}
properties.update(mapping)
submission = Submission(request.user, coordinator, request.fs, request.jt, properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
raise PopupException(_("Error submitting coordinator %s") % (coordinator,),
detail=ex._headers.get('oozie-error-message', ex))
def _submit_coordinator(request, coordinator, mapping):
try:
wf_doc = Document2.objects.get_by_uuid(user=request.user, uuid=coordinator.data['properties']['workflow'])
wf_dir = Submission(request.user, Workflow(document=wf_doc), request.fs, request.jt, mapping, local_tz=coordinator.data['properties']['timezone']).deploy()
properties = {'wf_application_path': request.fs.get_hdfs_path(wf_dir)}
properties.update(mapping)
submission = Submission(request.user, coordinator, request.fs, request.jt, properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
LOG.exception('Error submitting coordinator')
raise PopupException(_("Error submitting coordinator %s") % (coordinator,), detail=ex._headers.get('oozie-error-message', ex))
def submit_external_job(request, application_path):
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == "POST":
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param["name"], param["value"]) for param in params_form.cleaned_data])
mapping["dryrun"] = request.POST.get("dryrun_checkbox") == "on"
application_name = os.path.basename(application_path)
application_class = (
Bundle
if application_name == "bundle.xml"
else Coordinator
if application_name == "coordinator.xml"
else get_workflow()
)
mapping[application_class.get_application_path_key()] = application_path
try:
submission = Submission(request.user, fs=request.fs, jt=request.jt, properties=mapping)
job_id = submission.run(application_path)
except RestException, ex:
detail = ex._headers.get("oozie-error-message", ex)
if "Max retries exceeded with url" in str(detail):
detail = "%s: %s" % (_("The Oozie server is not running"), detail)
LOG.exception(smart_str(detail))
raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail)
request.info(_("Oozie job submitted"))
view = (
"list_oozie_bundle"
if application_name == "bundle.xml"
else "list_oozie_coordinator"
if application_name == "coordinator.xml"
else "list_oozie_workflow"
)
return redirect(reverse("oozie:%s" % view, kwargs={"job_id": job_id}))
else:
request.error(_("Invalid submission form: %s" % params_form.errors))
def _schedule_oozie_job(self, workspace_path, collection_name, input_path):
oozie = get_oozie(self.username)
properties = {
"dryrun": "False",
"zkHost": zkensemble(),
# these libs can be installed from here:
# https://drive.google.com/a/cloudera.com/folderview?id=0B1gZoK8Ae1xXc0sxSkpENWJ3WUU&usp=sharing
"oozie.libpath": CONFIG_INDEXER_LIBS_PATH.get(),
"security_enabled": "False",
"collectionName": collection_name,
"filePath": input_path,
"outputDir": "/user/%s/indexer" % self.username,
"workspacePath": workspace_path,
'oozie.wf.application.path': "${nameNode}%s" % workspace_path,
'user.name': self.username
}
submission = Submission(self.username, fs=self.fs, properties=properties)
job_id = submission.run(workspace_path)
return job_id
def _schedule_oozie_job(self, workspace_path, collection_name, input_path):
oozie = get_oozie(self.username)
properties = {
"dryrun": "False",
"zkHost": zkensemble(),
# these libs can be installed from here:
# https://drive.google.com/a/cloudera.com/folderview?id=0B1gZoK8Ae1xXc0sxSkpENWJ3WUU&usp=sharing
"oozie.libpath": CONFIG_INDEXER_LIBS_PATH.get(),
"security_enabled": "False",
"collectionName": collection_name,
"filePath": input_path,
"outputDir": "/user/%s/indexer" % self.username,
"workspacePath": workspace_path,
'oozie.wf.application.path': "${nameNode}%s" % workspace_path,
'user.name': self.username
}
submission = Submission(self.username,
fs=self.fs,
properties=properties)
job_id = submission.run(workspace_path)
return job_id
def _submit_coordinator(request, coordinator, mapping):
try:
wf = coordinator.workflow
if IS_MULTICLUSTER_ONLY.get() and has_multi_cluster():
mapping['auto-cluster'] = {
u'additionalClusterResourceTags': [],
u'automaticTerminationCondition':
u'EMPTY_JOB_QUEUE', #'u'NONE',
u'cdhVersion':
u'CDH514',
u'clouderaManagerPassword':
u'guest',
u'clouderaManagerUsername':
u'guest',
u'clusterName':
u'analytics4', # Add time variable
u'computeWorkersConfiguration': {
u'bidUSDPerHr': 0,
u'groupSize': 0,
u'useSpot': False
},
u'environmentName':
u'crn:altus:environments:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:environment:analytics/236ebdda-18bd-428a-9d2b-cd6973d42946',
u'instanceBootstrapScript':
u'',
u'instanceType':
u'm4.xlarge',
u'jobSubmissionGroupName':
u'',
u'jobs': [
{
u'failureAction': u'INTERRUPT_JOB_QUEUE',
u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
u'sparkJob': {
u'applicationArguments': ['5'],
u'jars': [
u's3a://datawarehouse-customer360/ETL/spark-examples.jar'
],
u'mainClass':
u'org.apache.spark.examples.SparkPi'
}
},
# {
# u'failureAction': u'INTERRUPT_JOB_QUEUE',
# u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
# u'sparkJob': {
# u'applicationArguments': ['10'],
# u'jars': [u's3a://datawarehouse-customer360/ETL/spark-examples.jar'],
# u'mainClass': u'org.apache.spark.examples.SparkPi'
# }
# },
# {
# u'failureAction': u'INTERRUPT_JOB_QUEUE',
# u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
# u'sparkJob': {
# u'applicationArguments': [u'filesystems3.conf'],
# u'jars': [u's3a://datawarehouse-customer360/ETL/envelope-0.6.0-SNAPSHOT-c6.jar'],
# u'mainClass': u'com.cloudera.labs.envelope.EnvelopeMain',
# u'sparkArguments': u'--archives=s3a://datawarehouse-customer360/ETL/filesystems3.conf'
# }
# }
],
u'namespaceName':
u'crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:analytics/7ea35fe5-dbc9-4b17-92b1-97a1ab32e410',
u'publicKey':
DEFAULT_PUBLIC_KEY.get(),
u'serviceType':
u'SPARK',
u'workersConfiguration': {},
u'workersGroupSize':
u'3'
}
wf_dir = Submission(
request.user,
wf,
request.fs,
request.jt,
mapping,
local_tz=coordinator.data['properties']['timezone']).deploy()
properties = {'wf_application_path': request.fs.get_hdfs_path(wf_dir)}
properties.update(mapping)
submission = Submission(request.user,
coordinator,
request.fs,
request.jt,
properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
LOG.exception('Error submitting coordinator')
raise PopupException(_("Error submitting coordinator %s") %
(coordinator, ),
detail=ex._headers.get('oozie-error-message', ex),
error_code=200)
def _submit_coordinator(request, coordinator, mapping):
try:
wf = coordinator.workflow
if IS_MULTICLUSTER_ONLY.get() and has_multi_cluster():
mapping['auto-cluster'] = {
u'additionalClusterResourceTags': [],
u'automaticTerminationCondition': u'EMPTY_JOB_QUEUE', #'u'NONE',
u'cdhVersion': u'CDH514',
u'clouderaManagerPassword': u'guest',
u'clouderaManagerUsername': u'guest',
u'clusterName': u'analytics4', # Add time variable
u'computeWorkersConfiguration': {
u'bidUSDPerHr': 0,
u'groupSize': 0,
u'useSpot': False
},
u'environmentName': u'crn:altus:environments:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:environment:analytics/236ebdda-18bd-428a-9d2b-cd6973d42946',
u'instanceBootstrapScript': u'',
u'instanceType': u'm4.xlarge',
u'jobSubmissionGroupName': u'',
u'jobs': [{
u'failureAction': u'INTERRUPT_JOB_QUEUE',
u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
u'sparkJob': {
u'applicationArguments': ['5'],
u'jars': [u's3a://datawarehouse-customer360/ETL/spark-examples.jar'],
u'mainClass': u'org.apache.spark.examples.SparkPi'
}
},
# {
# u'failureAction': u'INTERRUPT_JOB_QUEUE',
# u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
# u'sparkJob': {
# u'applicationArguments': ['10'],
# u'jars': [u's3a://datawarehouse-customer360/ETL/spark-examples.jar'],
# u'mainClass': u'org.apache.spark.examples.SparkPi'
# }
# },
# {
# u'failureAction': u'INTERRUPT_JOB_QUEUE',
# u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
# u'sparkJob': {
# u'applicationArguments': [u'filesystems3.conf'],
# u'jars': [u's3a://datawarehouse-customer360/ETL/envelope-0.6.0-SNAPSHOT-c6.jar'],
# u'mainClass': u'com.cloudera.labs.envelope.EnvelopeMain',
# u'sparkArguments': u'--archives=s3a://datawarehouse-customer360/ETL/filesystems3.conf'
# }
# }
],
u'namespaceName': u'crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:analytics/7ea35fe5-dbc9-4b17-92b1-97a1ab32e410',
u'publicKey': DEFAULT_PUBLIC_KEY.get(),
u'serviceType': u'SPARK',
u'workersConfiguration': {},
u'workersGroupSize': u'3'
}
wf_dir = Submission(request.user, wf, request.fs, request.jt, mapping, local_tz=coordinator.data['properties']['timezone']).deploy()
properties = {'wf_application_path': request.fs.get_hdfs_path(wf_dir)}
properties.update(mapping)
submission = Submission(request.user, coordinator, request.fs, request.jt, properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
LOG.exception('Error submitting coordinator')
raise PopupException(_("Error submitting coordinator %s") % (coordinator,), detail=ex._headers.get('oozie-error-message', ex), error_code=200)
