コード例 #1
0
def calculateGivenCombination(compilation_time, keywords, chrumprops, keys, combs, names, directory, plainName, name, idx):
		a = directory+'/default'
		b = directory+'/'+name
		shutil.copytree(a, b)
		
		substitutedWFPropertiesPath = directory+'/'+name+'/'+plainName 	
		substituteChrumWFProps(substitutedWFPropertiesPath,name,combs,keys,idx,compilation_time)
		
		execPath = sys.argv[0][:rightSignOrMax(sys.argv[0])]
#		sta = rightSignOrZero(sys.argv[3])
		wfname = 'workflow.xml'#sys.argv[3][sta:len(sys.argv[3])-len('.chrum')]
		sta = rightSignOrZero(sys.argv[2])
		finalWFPropertiesPath = directory+'/'+name+'/'+sys.argv[2][sta:len(sys.argv[2])-len('.chrum')]
#		finalWFPropertiesPath = directory+'/'+name+'/workflow.xml'

		wftxt = wf_transformations.main(execPath,substitutedWFPropertiesPath,sys.argv[3])
		substitutedWFXMLPath = directory+'/'+name+'/'+wfname
		f = open(substitutedWFXMLPath,'w')
		f.write(wftxt)
		f.close()
		
		hdfsProps = '/'.join([keywords['HDFS'],keywords['PROJECT'],compilation_time,name,plainName]) 
		hdfsPth = '/'.join([keywords['HDFS'],keywords['PROJECT'],compilation_time,name])
		hdfsSrc = '/'.join([keywords['HDFS'],keywords['PROJECT'],compilation_time,'default/*'])
		localWfSrc = directory+'/'+name+'/'+wfname
		subs = {'plain_name' : plainName, 
			'oozie_server' : keywords['OOZIE_SERVER'], 
			'oozie_port' : keywords['OOZIE_PORT'],
			'hdfs_wf_config' : hdfsProps,
			'hdfsPth' : hdfsPth,
			'hdfsSrc' : hdfsSrc,
			'localWfSrc' : localWfSrc,
			'substitutedWFPropertiesPath' : substitutedWFPropertiesPath,
			'finalWFPropertiesPath' : finalWFPropertiesPath,
			'substitutedWFXMLPath' : substitutedWFXMLPath
			}
  
		s = string.Template('\
#!/usr/bin/python \n\
import os	\n\
import stat	\n\
import time	\n\
import sys	\n\
\n\
#go to the executed script folder\n\
if sys.argv[0].rfind(\'/\') != -1:\n\
	path = sys.argv[0][:sys.argv[0].rfind(\'/\')]\n\
else:\n\
	path = os.getcwd()\n\
os.chdir(path)\n\
#perform cluster.properties substitution\n\
chrum_wf_props = \'/$substitutedWFPropertiesPath\'\n\
wf_props = \'/$substitutedWFXMLPath\'\n\
f = open(chrum_wf_props,\'r\')	\n\
txt = f.read()	\n\
f.close()	\n\
f = open(\'$finalWFPropertiesPath\',\'w\')	\n\
exec_time=str(time.time())	\n\
f.write(\'\
EXECUTION_TIME=\'+exec_time+\'\\n\
HDFS_EXEC_PATH=\\\n\
$hdfsPth/\'+exec_time+\'/\\n\'\
+txt)\n\
f.close()    \n\
#create folder on HDFS\n\
os.system(\'hadoop fs -mkdir $hdfsPth/\'+exec_time+\'/\')	\n\
#add all created files\n\
os.system(\'hadoop fs -put $localWfSrc  $hdfsPth/\'+exec_time+\'/\')	\n\
os.system(\'hadoop fs -cp $hdfsSrc $hdfsPth/\'+exec_time+\'/\')	\n\
#list all added files\n\
#os.system(\'hadoop fs -ls $hdfsPth/\'+exec_time+\'/\')	\n\
#run oozie workflow\n\
os.system(\'oozie job -oozie http://$oozie_server:$oozie_port/oozie -config $finalWFPropertiesPath -run\')	\n\
#os.system(\'oozie job -oozie http://$oozie_server:$oozie_port/oozie -config $hdfsPth/\'+exec_time+\'/cluster.properties -run\')	\n\
')
		f = open(directory+'/'+name+'/execute-in-oozie.py','w')
		tete = s.safe_substitute(subs);
		#print tete
		f.write(tete)
		f.close()
		os.system('chmod +x '+directory+'/'+name+'/execute-in-oozie.py')
コード例 #2
0
def main(compilation_time, keywords, chrumprops, keys, combs, names,
         directory):
    plainName = ''
    idx = sys.argv[2].rfind('/')
    if idx == -1:
        plainName = sys.argv[2]
    else:
        plainName = sys.argv[2][idx + 1:]

    for idx, name in enumerate(names):
        a = directory + '/default'
        b = directory + '/' + name
        shutil.copytree(a, b)

        f = open(directory + '/' + name + '/' + plainName, 'r')
        txt = 'PARAMETER_COMBINATION=' + name + '\n' + f.read()
        f.close()
        txt2 = []
        for li in txt.split('\n'):
            for inner_idx, val in enumerate(combs[idx]):
                if re.search('^@' + keys[inner_idx] + '@', li):
                    li = keys[inner_idx] + '=' + combs[idx][inner_idx]
            txt2.append(li)
        f = open(directory + '/' + name + '/' + plainName, 'w')
        f.write('\n'.join(txt2))
        f.close()

        sto = sys.argv[0].rfind('/')
        execPath = sys.argv[0][:sto]

        print sys.argv[1]  #''' CONF.CHRUM     '''
        print sys.argv[2]  #''' WF.CONF.PROPS  '''
        print sys.argv[3]  #''' WORKFLOW       '''
        #		main(, PropsPath, ChrumPath):

        wftxt = wf_transformations.main(
            sys.argv[3], directory + '/' + name + '/' + plainName, execPath)

        sta = sys.argv[3].rfind('/')
        wfname = sys.argv[3][sta:len(sys.argv[3]) - len('.chrum')]
        print directory + '/' + name + '/' + wfname
        f = open(directory + '/' + name + '/' + wfname, 'w')
        f.write(wftxt)
        f.close()

        localPropsTmp = '/'.join([
            keywords['HDFS'], keywords['PROJECT'], compilation_time, name,
            plainName
        ])
        hdfsProps = '/'.join([
            keywords['HDFS'], keywords['PROJECT'], compilation_time, name,
            plainName
        ])
        hdfsPth = '/'.join(
            [keywords['HDFS'], keywords['PROJECT'], compilation_time, name])

        subs = {
            'plain_name': plainName,
            'oozie_server': keywords['OOZIE_SERVER'],
            'oozie_port': keywords['OOZIE_PORT'],
            'hdfs_wf_config': hdfsProps,
            'hdfsPth': hdfsPth
        }

        s = string.Template('\
#!/usr/bin/python \n\
import os	\n\
import time	\n\
import sys	\n\
\n\
path = sys.argv[0][:sys.argv[0].rfind(\'/\')]\n\
os.chdir(path)\n\
chrum_wf_props = os.getcwd()+\'/cluster.properties.chrum\'\n\
wf_props = os.getcwd()+\'/cluster.properties\'\n\
f = open(chrum_wf_props,\'r\')	\n\
txt = f.read()	\n\
f.close()	\n\
f = open(wf_props,\'w\')	\n\
exec_time=str(time.time())	\n\
f.write(\'\
EXECUTION_TIME=\'+exec_time+\'\\n\
HDFS_EXEC_PATH=\\\n\
$hdfsPth/\'+exec_time+\'/\\n\'\
+txt)\n\
f.close()    \n\
\'\'\'	\n\
cp -r ../pig/*.pig ${WORKFLOW_LOCAL_PIG_DIR}	\n\
echo "Recreating workflow data in HDFS"	\n\
hadoop fs -rm -r ${WORKFLOW_HDFS_DIR}	\n\
hadoop fs -mkdir ${WORKFLOW_HDFS_DIR}	\n\
echo "Putting current workflow data to HDFS"	\n\
hadoop fs -put ${TASK}/* ${WORKFLOW_HDFS_DIR}	\n\
\'\'\'	\n\
\'\'\'	\n\
os.system(\'hadoop fs -put \'+wf_props+\'$hdfsPth/\'+exec_time+\'/\')	\n\
\n\
os.system(\'cp os.getcwd()+\'/\'$plain_name $hadoop_config)	\n\
os.system(\'oozie job -oozie http://$oozie_server:$oozie_port/oozie -config $hdfs_wf_config -run\')	\n\
os.system(\'oozie job -oozie http://$oozie_server:$oozie_port/oozie -config $hdfs_wf_config -run\')	\n\
\'\'\'	\n\
')
        f = open(directory + '/' + name + '/execute-in-oozie.py', 'w')
        tete = s.safe_substitute(subs)
        print tete
        f.write(tete)
        f.close()