def restoreDeepEval(): command = [] command.append('mkdir -p /mnt/data/tmp/DEEPEVAL') command.append('cd /mnt/data/tmp/DEEPEVAL') listEval = [ 'DEEPEVAL-default-2016-03-18.tar.gz', 'DEEPEVAL-fusion1-2016-03-18.tar.gz', # 'DEEPEVAL-fusion16cnn-2016-03-18.tar.gz', # 'DEEPEVAL-fusion16cnn2-2016-03-18.tar.gz', # 'DEEPEVAL-fusion8cnn-2016-03-18.tar.gz', 'DEEPEVAL-norm51-2016-03-18.tar.gz', 'DEEPEVAL-norm51Scale1-2016-03-18.tar.gz', 'DEEPEVAL-norm51Scale2-2016-03-18.tar.gz', 'DEEPEVAL-norm51Scale3-2016-03-18.tar.gz', 'DEEPEVAL-ratio-2016-03-18.tar.gz', 'DEEPEVAL-veac-2016-03-18.tar.gz'] for l in listEval: command.append('gsutil cp gs://deep-store/{0} .'.format(l)) command.append('tar -xf {0}'.format(l)) command.append('rm -r {0}'.format(l)) # command.append('gsutil cp gs://deep-store/DEEPEVAL.tar.gz .') # command.append('rm -rf DEEPEVAL/fusion9a') # command.append('rsync -la DEEPEVAL/* /mnt/data/tmp/DEEPEVAL/') cluster.runOnAllNodesAsync(';'.join(command))
def setupSoftFiles(): """Copy hdfs soft from common data disk to homedir """ listCommand = [] listCommand.append('gsutil cp {0} .'.format(bucketstore)) listCommand.append('tar -xf {0}'.format(os.path.basename(bucketstore))) command = ';'.join(listCommand) cluster.runOnAllNodesAsync(command)
def setupPostinstall(): listCommand = [] listCommand.append('cd {0}'.format(softdir)) listCommand.append(postinstallscript) listCommand.append('ln -fs {0} {1}'.format(pythonbin, spark.pathpython)) listCommand = ';'.join(listCommand) cluster.runOnAllNodesAsync(listCommand) addjars()
def restoreFromStore(): listCommand = [] listCommand.append('gsutil cp {0} {1}'.format(bucketstore, os.path.dirname(softdir))) listCommand.append('cd {0}'.format(os.path.dirname(softdir))) listCommand.append('tar -xf {0}'.format(os.path.basename(bucketstore))) command = ';'.join(listCommand) cluster.runOnAllNodesAsync(command)
def setupBashrc(): inStartup = utils.getTemplateFile('hdfs-setup_hdfs.sh') outStartup = utils.getLocalTempFile('setup_hdfs.sh') utils.stringReplaceInFile( inStartup, outStartup, { 'XXHADOOPLOCALPATHXX': "'{0}'".format(softdir) }) cluster.rsyncOnAllNodesLocalhostToLocalAsync(outStartup, '~/setup_hdfs.sh') command = 'echo "source setup_hdfs.sh" >> setup.sh' cluster.runOnAllNodesAsync(command)
def setupProfile(): inStartup = utils.getTemplateFile('package-setup_profile_package.sh') outStartup = utils.getLocalTempFile('setup_profile_package.sh') utils.stringReplaceInFile( inStartup, outStartup, { # 'XX-PYTHON-XX': "'{0}'".format(pythonbin), 'XX-SOURCESCRIPT-XX': sourcescript }) cluster.rsyncOnAllNodesLocalhostToLocalAsync( outStartup, '~/setup_profile_package.sh') command = 'echo "source $HOME/setup_profile_package.sh" >> setup_profile.sh' cluster.runOnAllNodesAsync(command)
def setupBashrc(): inStartup = utils.getTemplateFile('spark-setup_spark.sh') outStartup = utils.getLocalTempFile('setup_spark.sh') utils.stringReplaceInFile( inStartup, outStartup, { 'XXSPARKLOCALPATHXX': "'{0}'".format(softdir) }) cluster.rsyncOnAllNodesLocalhostToLocalAsync( outStartup, '~/setup_spark.sh') command = 'echo "source setup_spark.sh" >> setup.sh' cluster.runOnAllNodesAsync(command) if useCloudConnector and cluster.getNSlaves() > 0: addjars('{0}/lib/gcs-connector-latest-hadoop2.jar'.format(softdir))
def addjars(jars): try: jars.split(':') jars = [jars] except: pass for j in jars: cluster.runOnMaster("echo '' >> setup_spark.sh".format(j)) addJar = [] addJar.append('if [[ -z "$SPARKJARS" ]]; then ') addJar.append(' export SPARKJARS={0}'.format(j)) addJar.append('else ') addJar.append(' export SPARKJARS={0},$SPARKJARS'.format(j)) addJar.append('fi') c = ["echo '" + a + "' >> setup_spark.sh" for a in addJar] print c cluster.runOnAllNodesAsync(';'.join(c)) jars = ':'.join(jars) command = [] command.append( 'echo "" >> {0}'.format(softdir + '/conf/spark-defaults.conf')) command.append('echo "spark.driver.extraClassPath {0}" >> {1}'.format( jars, softdir + '/conf/spark-defaults.conf')) command.append('echo "spark.executor.extraClassPath {0}" >> {1}'.format( jars, softdir + '/conf/spark-defaults.conf')) command = ';'.join(command) cluster.runOnAllNodesAsync(command)
def restoreDeepModels(): """Download and install models from bucket """ command = [] command.append('mkdir /mnt/data/tmp/DEEPSTORE/') command.append('rm -r /mnt/data/tmp/DEEPSTORE/*') command.append('cd /mnt/data/tmp/') command.append('mkdir download') command.append('cd download') listStore = [ 'deepStore-default-2016-03-18.tar.gz', 'deepStore-fusion1-2016-03-18.tar.gz', 'deepStore-fusion16cnn-2016-03-18.tar.gz', 'deepStore-fusion16cnn2-2016-03-18.tar.gz', 'deepStore-fusion8cnn-2016-03-18.tar.gz', 'deepStore-fusion9a-2016-03-18.tar.gz', 'deepStore-norm51-2016-03-18.tar.gz', 'deepStore-norm51Scale1-2016-03-18.tar.gz', 'deepStore-norm51Scale2-2016-03-18.tar.gz', 'deepStore-norm51Scale3-2016-03-18.tar.gz', 'deepStore-ratio-2016-03-18.tar.gz', 'deepStore-veac-2016-03-18.tar.gz'] for l in listStore: command.append('gsutil cp gs://deep-store/{0} .'.format(l)) command.append('tar -zxvf {0}'.format(l)) command.append('rm {0}'.format(l)) command.append('gsutil cp gs://deep-store/deepStore* .') command.append('mv * /mnt/data/tmp/DEEPSTORE/') command = ';'.join(command) cluster.runOnAllNodesAsync(command)
def setupConfigurationFiles(): """Deploy hadoop""" (listnodes, nodes) = cluster.instanceListAll() mastername = listnodes['master'][0] inCoreSite = utils.getTemplateFile('hdfs-core-site.xml') outCoreSite = utils.getLocalTempFile('core-site.xml') inHdfsSite = utils.getTemplateFile('hdfs-hdfs-site.xml') outHdfsSite = utils.getLocalTempFile('hdfs-site.xml') outSlave = utils.getLocalTempFile('slaves') print '[ Configuring Hadoop ]' utils.stringReplaceInFile( inCoreSite, outCoreSite, { 'PUT-MASTER-IP': mastername, 'XX-PROJECTID-XX': utils.getProjectProperties()['Project'] }) pathnamenode = os.path.join(datadir, 'namenode') pathdatanode = os.path.join(datadir, 'datanode') utils.stringReplaceInFile( inHdfsSite, outHdfsSite, { 'XXREPLICATIONXX': '3', 'XXNAMENODEXX': pathnamenode, 'XXDATANODEXX': pathdatanode, }) with open(outSlave, 'w') as streamOut: for namenode in listnodes['slaves']: streamOut.write(namenode + '\n') cluster.rsyncOnAllNodesLocalhostToLocalAsync( outCoreSite, softdir + '/etc/hadoop/core-site.xml') cluster.rsyncOnAllNodesLocalhostToLocalAsync( outHdfsSite, softdir + '/etc/hadoop/hdfs-site.xml') cluster.rsyncOnAllNodesLocalhostToLocalAsync( outSlave, softdir + '/etc/hadoop/slaves') if useCloudConnector: cluster.runOnAllNodesAsync('gsutil cp {0} .'.format(bucketconnector)) # cluster.runOnAllNodesAsync('wget https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-latest-hadoop2.jar') cluster.runOnAllNodesAsync( 'cp gcs-connector-latest-hadoop2.jar {0}/share/hadoop/common/'.format(softdir)) listCommand = [] listCommand.append("mkdir -p {0}".format(pathnamenode)) listCommand.append("mkdir -p {0}".format(pathdatanode)) listCommand.append("chmod -R a+rwx {0}".format(pathnamenode)) listCommand.append("chmod -R a+rwx {0}".format(pathdatanode)) command = ';'.join(listCommand) cluster.runOnAllNodesAsync(command)
def additionPython(): cluster.runOnMasterX("pip install ipdb") cluster.runOnAllNodesAsync('ln -s /mnt/data/package/softs/pypilok/irt')
def additionPython(): cluster.runOnAllNodesAsync('ln -s /mnt/data/package/softs/pypilok/irt') cluster.runOnAllNodesAsync('ln -s /mnt/data/package/softs/pypilok/ads')
def syncEvalFromMaster(): command = 'rsync -la cluster-data-master:/mnt/data/tmp/DEEPEVAL /mnt/data/tmp/' cluster.runOnAllNodesAsync(command)
def setupConfigurationFiles(): """Deploy spark configuration files""" (listnodes, nodes) = cluster.instanceListAll() # We create here a fake python link. # This python is used as the main spark driver # If we need to change the spark python driver, # we just have to overwrite this link. cluster.runOnAllNodesAsync('ln -fs `which python` {0}'.format(pathpython)) # ------------------------------------------- # handling of slaves # ------------------------------------------- # The slave file contains information # about which hosts have to be used outSlave = utils.getLocalTempFile('slaves') with open(outSlave, 'w') as streamOut: for namenode in listnodes['slaves']: streamOut.write(namenode + '\n') # ------------------------------------------- # handling of spark configuration # ------------------------------------------- if cluster.getNSlaves() > 0: sparkMaster = 'spark://{0}:7077'.format(cluster.getMasterName()) else: sparkMaster = 'local[{0}]'.format((cluster.nCores - 1)) inConf = utils.getTemplateFile('spark-spark-defaults.conf') outConf = utils.getLocalTempFile('spark-defaults.conf') maxSlaves = int(cluster.nCores - 1) utils.stringReplaceInFile( inConf, outConf, { 'XX-DRIVER-MEM-XX': mastermemory, 'XX-EXECUTOR-MEM-XX': executormemory, 'XX-SPARKMASTER-XX': sparkMaster, 'XX-LOCAL-DIR-XX': localtempdir, 'XX-CORES-XX': '{0}'.format(maxSlaves) }) inEnv = utils.getTemplateFile('spark-spark-env.sh') outEnv = utils.getLocalTempFile('spark-env.sh') utils.stringReplaceInFile( inEnv, outEnv, { 'XX-PYSPARK_PYTHON-XX': '"{0}"'.format(pathpython), 'XX-SPARKMASTER-XX': sparkMaster, 'XX-PYSPARK_DRIVER_PYTHON-XX': '"{0}"'.format(pathpython), 'XX-PYTHONPATH-XX': '"{0}"'.format(pythonpath), 'XX-LOCAL-DIR-XX': localtempdir, 'XX-MASTER-IP-XX': '"{0}"'.format(cluster.ipGetMaster()) }) cluster.rsyncOnAllNodesLocalhostToLocalAsync( outSlave, softdir + '/conf/slaves') cluster.rsyncOnAllNodesLocalhostToLocalAsync( outConf, softdir + '/conf/spark-defaults.conf') cluster.rsyncOnAllNodesLocalhostToLocalAsync( outEnv, softdir + '/conf/spark-env.sh') cluster.runOnAllNodesAsync('mkdir -p /tmp/spark-events') # we remove info level display from spark... cluster.runOnMaster( 'sed -i "s/log4j.rootCategory=INFO/log4j.rootCategory=WARN/g" {0}/conf/log4j.properties.template'.format(softdir)) cluster.runOnMaster( 'cp {0}/conf/log4j.properties.template {0}/conf/log4j.properties'.format(softdir)) # -------------------------------- # handling of connector # -------------------------------- # We install here the hadoop connector for google cloud storage # This connector permits writing data on google cs directly # from spark if useCloudConnector and cluster.getNSlaves() > 0: cluster.runOnAllNodesAsync('gsutil cp {0} .'.format(bucketconnector)) # cluster.runOnAllNodesAsync('wget https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-latest-hadoop2.jar') cluster.runOnAllNodesAsync( 'cp gcs-connector-latest-hadoop2.jar {0}/lib/'.format(softdir)) cluster.runOnAllNodesAsync( 'cp {0}/etc/hadoop/core-site.xml {1}/conf/'.format(hadoop.softdir, softdir)) # ------------------------------------------ # Deployment of spark overloading scripts # ------------------------------------------ # # One problem with spark in standalone mode, is that # we have to use the client mode. # with the client mode, we cannot use the spark default conf # for setting additional jars at launch. # # We therefore use two scripts, one for spark-submit, one for pyspark # for overloading the calls. # # These scripts tests for the existence of jar variables # and make the call accordingly inPyspark = utils.getTemplateFile('pyspark-jars') cluster.rsyncOnAllNodesLocalhostToLocalAsync(inPyspark, 'pyspark-jars') inSubmit = utils.getTemplateFile('spark-submit-jars') cluster.rsyncOnAllNodesLocalhostToLocalAsync(inSubmit, 'spark-submit-jars')