def calculateGivenCombination(compilation_time, keywords, chrumprops, keys, combs, names, directory, plainName, name, idx):
a = directory+'/default'
b = directory+'/'+name
shutil.copytree(a, b)
substitutedWFPropertiesPath = directory+'/'+name+'/'+plainName
substituteChrumWFProps(substitutedWFPropertiesPath,name,combs,keys,idx,compilation_time)
execPath = sys.argv[0][:rightSignOrMax(sys.argv[0])]
# sta = rightSignOrZero(sys.argv[3])
wfname = 'workflow.xml'#sys.argv[3][sta:len(sys.argv[3])-len('.chrum')]
sta = rightSignOrZero(sys.argv[2])
finalWFPropertiesPath = directory+'/'+name+'/'+sys.argv[2][sta:len(sys.argv[2])-len('.chrum')]
# finalWFPropertiesPath = directory+'/'+name+'/workflow.xml'
wftxt = wf_transformations.main(execPath,substitutedWFPropertiesPath,sys.argv[3])
substitutedWFXMLPath = directory+'/'+name+'/'+wfname
f = open(substitutedWFXMLPath,'w')
f.write(wftxt)
f.close()
hdfsProps = '/'.join([keywords['HDFS'],keywords['PROJECT'],compilation_time,name,plainName])
hdfsPth = '/'.join([keywords['HDFS'],keywords['PROJECT'],compilation_time,name])
hdfsSrc = '/'.join([keywords['HDFS'],keywords['PROJECT'],compilation_time,'default/*'])
localWfSrc = directory+'/'+name+'/'+wfname
subs = {'plain_name' : plainName,
'oozie_server' : keywords['OOZIE_SERVER'],
'oozie_port' : keywords['OOZIE_PORT'],
'hdfs_wf_config' : hdfsProps,
'hdfsPth' : hdfsPth,
'hdfsSrc' : hdfsSrc,
'localWfSrc' : localWfSrc,
'substitutedWFPropertiesPath' : substitutedWFPropertiesPath,
'finalWFPropertiesPath' : finalWFPropertiesPath,
'substitutedWFXMLPath' : substitutedWFXMLPath
}
s = string.Template('\
#!/usr/bin/python \n\
import os \n\
import stat \n\
import time \n\
import sys \n\
\n\
#go to the executed script folder\n\
if sys.argv[0].rfind(\'/\') != -1:\n\
path = sys.argv[0][:sys.argv[0].rfind(\'/\')]\n\
else:\n\
path = os.getcwd()\n\
os.chdir(path)\n\
#perform cluster.properties substitution\n\
chrum_wf_props = \'/$substitutedWFPropertiesPath\'\n\
wf_props = \'/$substitutedWFXMLPath\'\n\
f = open(chrum_wf_props,\'r\') \n\
txt = f.read() \n\
f.close() \n\
f = open(\'$finalWFPropertiesPath\',\'w\') \n\
exec_time=str(time.time()) \n\
f.write(\'\
EXECUTION_TIME=\'+exec_time+\'\\n\
HDFS_EXEC_PATH=\\\n\
$hdfsPth/\'+exec_time+\'/\\n\'\
+txt)\n\
f.close() \n\
#create folder on HDFS\n\
os.system(\'hadoop fs -mkdir $hdfsPth/\'+exec_time+\'/\') \n\
#add all created files\n\
os.system(\'hadoop fs -put $localWfSrc $hdfsPth/\'+exec_time+\'/\') \n\
os.system(\'hadoop fs -cp $hdfsSrc $hdfsPth/\'+exec_time+\'/\') \n\
#list all added files\n\
#os.system(\'hadoop fs -ls $hdfsPth/\'+exec_time+\'/\') \n\
#run oozie workflow\n\
os.system(\'oozie job -oozie http://$oozie_server:$oozie_port/oozie -config $finalWFPropertiesPath -run\') \n\
#os.system(\'oozie job -oozie http://$oozie_server:$oozie_port/oozie -config $hdfsPth/\'+exec_time+\'/cluster.properties -run\') \n\
')
f = open(directory+'/'+name+'/execute-in-oozie.py','w')
tete = s.safe_substitute(subs);
#print tete
f.write(tete)
f.close()
os.system('chmod +x '+directory+'/'+name+'/execute-in-oozie.py')
def main(compilation_time, keywords, chrumprops, keys, combs, names,
directory):
plainName = ''
idx = sys.argv[2].rfind('/')
if idx == -1:
plainName = sys.argv[2]
else:
plainName = sys.argv[2][idx + 1:]
for idx, name in enumerate(names):
a = directory + '/default'
b = directory + '/' + name
shutil.copytree(a, b)
f = open(directory + '/' + name + '/' + plainName, 'r')
txt = 'PARAMETER_COMBINATION=' + name + '\n' + f.read()
f.close()
txt2 = []
for li in txt.split('\n'):
for inner_idx, val in enumerate(combs[idx]):
if re.search('^@' + keys[inner_idx] + '@', li):
li = keys[inner_idx] + '=' + combs[idx][inner_idx]
txt2.append(li)
f = open(directory + '/' + name + '/' + plainName, 'w')
f.write('\n'.join(txt2))
f.close()
sto = sys.argv[0].rfind('/')
execPath = sys.argv[0][:sto]
print sys.argv[1] #''' CONF.CHRUM '''
print sys.argv[2] #''' WF.CONF.PROPS '''
print sys.argv[3] #''' WORKFLOW '''
# main(, PropsPath, ChrumPath):
wftxt = wf_transformations.main(
sys.argv[3], directory + '/' + name + '/' + plainName, execPath)
sta = sys.argv[3].rfind('/')
wfname = sys.argv[3][sta:len(sys.argv[3]) - len('.chrum')]
print directory + '/' + name + '/' + wfname
f = open(directory + '/' + name + '/' + wfname, 'w')
f.write(wftxt)
f.close()
localPropsTmp = '/'.join([
keywords['HDFS'], keywords['PROJECT'], compilation_time, name,
plainName
])
hdfsProps = '/'.join([
keywords['HDFS'], keywords['PROJECT'], compilation_time, name,
plainName
])
hdfsPth = '/'.join(
[keywords['HDFS'], keywords['PROJECT'], compilation_time, name])
subs = {
'plain_name': plainName,
'oozie_server': keywords['OOZIE_SERVER'],
'oozie_port': keywords['OOZIE_PORT'],
'hdfs_wf_config': hdfsProps,
'hdfsPth': hdfsPth
}
s = string.Template('\
#!/usr/bin/python \n\
import os \n\
import time \n\
import sys \n\
\n\
path = sys.argv[0][:sys.argv[0].rfind(\'/\')]\n\
os.chdir(path)\n\
chrum_wf_props = os.getcwd()+\'/cluster.properties.chrum\'\n\
wf_props = os.getcwd()+\'/cluster.properties\'\n\
f = open(chrum_wf_props,\'r\') \n\
txt = f.read() \n\
f.close() \n\
f = open(wf_props,\'w\') \n\
exec_time=str(time.time()) \n\
f.write(\'\
EXECUTION_TIME=\'+exec_time+\'\\n\
HDFS_EXEC_PATH=\\\n\
$hdfsPth/\'+exec_time+\'/\\n\'\
+txt)\n\
f.close() \n\
\'\'\' \n\
cp -r ../pig/*.pig ${WORKFLOW_LOCAL_PIG_DIR} \n\
echo "Recreating workflow data in HDFS" \n\
hadoop fs -rm -r ${WORKFLOW_HDFS_DIR} \n\
hadoop fs -mkdir ${WORKFLOW_HDFS_DIR} \n\
echo "Putting current workflow data to HDFS" \n\
hadoop fs -put ${TASK}/* ${WORKFLOW_HDFS_DIR} \n\
\'\'\' \n\
\'\'\' \n\
os.system(\'hadoop fs -put \'+wf_props+\'$hdfsPth/\'+exec_time+\'/\') \n\
\n\
os.system(\'cp os.getcwd()+\'/\'$plain_name $hadoop_config) \n\
os.system(\'oozie job -oozie http://$oozie_server:$oozie_port/oozie -config $hdfs_wf_config -run\') \n\
os.system(\'oozie job -oozie http://$oozie_server:$oozie_port/oozie -config $hdfs_wf_config -run\') \n\
\'\'\' \n\
')
f = open(directory + '/' + name + '/execute-in-oozie.py', 'w')
tete = s.safe_substitute(subs)
print tete
f.write(tete)
f.close()
