def createPipeline(host,
clusterName,
userName,
pipelineName,
protocol,
queue,
config,
pipelineId=None,
taskName=None,
parentPipeline=None,
timeout=30,
tries=4):
params = dict(cluster=clusterName,
user_name=userName,
pipeline_name=pipelineName,
protocol=protocol,
queue=queue,
config=config)
if pipelineId is not None:
params['pipeline_id'] = pipelineId
if taskName is not None:
params['task_name'] = taskName
if parentPipeline:
params['parent_pipeline'] = parentPipeline
return http.performQuery(host,
CREATE_URL,
params,
timeout=timeout,
tries=tries)
def tagData(host,
clusterName,
userName,
action,
tagName,
files,
metadata,
recursive,
expand,
compressDir,
urls=[],
timeout=30,
tries=4):
return http.performQuery(host,
CREATEUPDATE_URL,
dict(cluster=clusterName,
user_name=userName,
tag_name=tagName,
action=action,
files=files,
metadata=metadata,
recursive=recursive,
expand=expand,
compress_dir=compressDir,
urls=urls),
timeout=timeout,
tries=tries)
def updateConfigPipeline(host, clusterName, userName, criteria, config):
return http.performQuery(
host, UPDATECONFIG_URL,
dict(cluster=clusterName,
user_name=userName,
criteria=criteria,
config=config))
def loadTagsBy(host, clusterName, userName, criteria, detail):
return http.performQuery(host,
LIST_URL,
dict(cluster=clusterName,
user_name=userName,
criteria=criteria,
detail=detail))
def createPipeline(
host,
clusterName,
userName,
pipelineName,
protocol,
queue,
config,
pipelineId=None,
taskName=None,
parentPipeline=None,
timeout=30,
tries=4,
):
params = dict(
cluster=clusterName,
user_name=userName,
pipeline_name=pipelineName,
protocol=protocol,
queue=queue,
config=config,
)
if pipelineId is not None:
params["pipeline_id"] = pipelineId
if taskName is not None:
params["task_name"] = taskName
if parentPipeline:
params["parent_pipeline"] = parentPipeline
return http.performQuery(host, CREATE_URL, params, timeout=timeout, tries=tries)
def pipelineListBy(host, clusterName, userName, criteria, detail):
return http.performQuery(
host, LIST_URL,
dict(cluster=clusterName,
user_name=userName,
criteria=criteria,
detail=detail))
def tagData(host,
clusterName,
userName,
action,
tagName,
files,
metadata,
recursive,
expand,
compressDir,
urls=[],
timeout=30,
tries=4):
return http.performQuery(host,
CREATEUPDATE_URL,
dict(cluster=clusterName,
user_name=userName,
tag_name=tagName,
action=action,
files=files,
metadata=metadata,
recursive=recursive,
expand=expand,
compress_dir=compressDir,
urls=urls),
timeout=timeout,
tries=tries)
def terminateInstances(host, clusterName, userName, byCriteria, criteriaValues, timeout=30, tries=4):
return http.performQuery(host,
TERMINATEINSTANCES_URL,
dict(cluster_name=clusterName,
user_name=userName,
by_criteria=byCriteria,
criteria_values=criteriaValues),
timeout=timeout,
tries=tries)
def addInstances(host, clusterName, userName, numExec, numData, timeout=30, tries=4):
return http.performQuery(host,
ADDINSTANCES_URL,
dict(cluster_name=clusterName,
user_name=userName,
num_exec=numExec,
num_data=numData),
timeout=timeout,
tries=tries)
def saveCredential(host, clusterName, credName, description, ctype, cert, pkey, metadata, conf):
return http.performQuery(host,
SAVECREDENTIAL_URL,
dict(cluster=clusterName,
credential_name=credName,
description=description,
ctype=ctype,
cert=cert,
pkey=pkey,
metadata=metadata,
conf=conf))
def loadTag(host, clusterName, userName, tagName):
def _failIfNoTag(tags):
if not tags or len(tags) > 1:
raise persist.TagNotFoundError(tagName)
else:
return tags[0]
return http.performQuery(host,
LIST_URL,
dict(cluster=clusterName,
user_name=userName,
criteria={'tag_name': tagName},
detail=True)).addCallback(_failIfNoTag)
def listClusters(host, criteria, userName, authToken=None, timeout=30, tries=4):
options = {}
if authToken:
options['auth_token'] = authToken
return http.performQuery(host,
LISTCLUSTERS_URL,
dict(criteria=criteria,
user_name=userName,
options=options),
timeout=timeout,
tries=tries)
def loadTag(host, clusterName, userName, tagName):
def _failIfNoTag(tags):
if not tags or len(tags) > 1:
raise persist.TagNotFoundError(tagName)
else:
return tags[0]
return http.performQuery(
host, LIST_URL,
dict(cluster=clusterName,
user_name=userName,
criteria={'tag_name': tagName},
detail=True)).addCallback(_failIfNoTag)
def terminateCluster(host,
clusterName,
userName,
authToken,
timeout=30,
tries=4):
return http.performQuery(host,
TERMINATECLUSTER_URL,
dict(cluster_name=clusterName,
user_name=userName,
auth_token=authToken),
timeout=timeout,
tries=tries)
def terminateCluster(host,
clusterName,
userName,
authToken,
timeout=30,
tries=4):
return http.performQuery(host,
TERMINATECLUSTER_URL,
dict(cluster_name=clusterName,
user_name=userName,
auth_token=authToken),
timeout=timeout,
tries=tries)
def terminateInstances(host,
clusterName,
userName,
byCriteria,
criteriaValues,
timeout=30,
tries=4):
return http.performQuery(host,
TERMINATEINSTANCES_URL,
dict(cluster_name=clusterName,
user_name=userName,
by_criteria=byCriteria,
criteria_values=criteriaValues),
timeout=timeout,
tries=tries)
def addInstances(host,
clusterName,
userName,
numExec,
numData,
timeout=30,
tries=4):
return http.performQuery(host,
ADDINSTANCES_URL,
dict(cluster_name=clusterName,
user_name=userName,
num_exec=numExec,
num_data=numData),
timeout=timeout,
tries=tries)
def _(request):
if request.body['cluster'] == 'local':
defer.returnValue(request)
else:
clusters = yield clusters_client.listClusters({'cluster_name': request.body['cluster']},
request.body['user_name'])
cluster = clusters[0]
ret = yield http.performQuery(cluster['master']['public_dns'],
url,
func.updateDict(request.body, {'cluster': 'local'}),
timeout=10,
tries=3)
defer_pipe.emit(request.update(response=ret))
def realizePhantom(host,
clusterName,
userName,
tagName,
phantom,
metadata,
timeout=30,
tries=4):
return http.performQuery(host,
REALIZE_URL,
dict(cluster=clusterName,
user_name=userName,
tag_name=tagName,
phantom=phantom,
metadata=metadata),
timeout=timeout,
tries=tries)
def listClusters(host,
criteria,
userName,
authToken=None,
timeout=30,
tries=4):
options = {}
if authToken:
options['auth_token'] = authToken
return http.performQuery(host,
LISTCLUSTERS_URL,
dict(criteria=criteria,
user_name=userName,
options=options),
timeout=timeout,
tries=tries)
def realizePhantom(host,
clusterName,
userName,
tagName,
phantom,
metadata,
timeout=30,
tries=4):
return http.performQuery(host,
REALIZE_URL,
dict(cluster=clusterName,
user_name=userName,
tag_name=tagName,
phantom=phantom,
metadata=metadata),
timeout=timeout,
tries=tries)
def runPipeline(
host, clusterName, userName, parentPipeline, bareRun, queue, config, overwrite=False, timeout=30, tries=4
):
return http.performQuery(
host,
RUN_URL,
dict(
cluster=clusterName,
user_name=userName,
parent_pipeline=parentPipeline,
bare_run=bareRun,
overwrite=overwrite,
queue=queue,
config=config,
),
timeout=timeout,
tries=tries,
)
def startCluster(host,
clusterName,
userName,
numExec,
numData,
credName,
conf,
timeout=30,
tries=4):
return http.performQuery(host,
STARTCLUSTER_URL,
dict(cluster_name=clusterName,
user_name=userName,
num_exec=numExec,
num_data=numData,
cred_name=credName,
conf=conf),
timeout=timeout,
tries=tries)
def _(request):
if request.body['cluster'] == 'local':
defer.returnValue(request)
else:
clusters = yield clusters_client.listClusters(
{'cluster_name': request.body['cluster']},
request.body['user_name'])
cluster = clusters[0]
ret = yield http.performQuery(cluster['master']['public_dns'],
url,
func.updateDict(
request.body,
{'cluster': 'local'}),
timeout=10,
tries=3)
defer_pipe.emit(request.update(response=ret))
def startCluster(host,
clusterName,
userName,
numExec,
numData,
credName,
conf,
timeout=30,
tries=4):
return http.performQuery(host,
STARTCLUSTER_URL,
dict(cluster_name=clusterName,
user_name=userName,
num_exec=numExec,
num_data=numData,
cred_name=credName,
conf=conf),
timeout=timeout,
tries=tries)
def runPipeline(host,
clusterName,
userName,
parentPipeline,
bareRun,
queue,
config,
overwrite=False,
timeout=30,
tries=4):
return http.performQuery(host,
RUN_URL,
dict(cluster=clusterName,
user_name=userName,
parent_pipeline=parentPipeline,
bare_run=bareRun,
overwrite=overwrite,
queue=queue,
config=config),
timeout=timeout,
tries=tries)
def resumePipeline(host, clusterName, userName, pipelineName):
return http.performQuery(
host, RESUME_URL, dict(cluster=clusterName, user_name=userName, pipeline_name=pipelineName)
)
def resumePipeline(host, clusterName, userName, pipelineName):
return http.performQuery(
host, RESUME_URL,
dict(cluster=clusterName,
user_name=userName,
pipeline_name=pipelineName))
def loadTask(host, clusterName, userName, taskName):
return http.performQuery(host,
TASK_URL,
dict(cluster=clusterName,
user_name=userName,
task_name=taskName)).addCallback(lambda r : r[0])
def updateConfigPipeline(host, clusterName, userName, criteria, config):
return http.performQuery(
host, UPDATECONFIG_URL, dict(cluster=clusterName, user_name=userName, criteria=criteria, config=config)
)
def loadTask(host, clusterName, userName, taskName):
return http.performQuery(
host, TASK_URL,
dict(cluster=clusterName, user_name=userName,
task_name=taskName)).addCallback(lambda r: r[0])
def describeCredentials(host, clusterName, credentials):
return http.performQuery(host,
SAVECREDENTIAL_URL,
dict(cluster=clusterName,
credential_names=credentials))
def listCredentials(host, clusterName):
return http.performQuery(host,
SAVECREDENTIAL_URL,
dict(cluster=clusterName))