def getZipFile(cls, version=HBase.getVersionFromBuild(), isRU=False): # download for linux, no download for windows HBASE_VER_BUILD = version if Machine.isWindows(): zipFile = os.path.join( Config.get('slider', 'SLIDER_HOME'), "app-packages", "slider-hbase-app-win-package-%s.zip" % HBASE_VER_BUILD) return zipFile pkg_list = "pkg-list_qe.txt" path = os.path.join(Config.getEnv('ARTIFACTS_DIR'), pkg_list) #pkgUrl = Config.get('slider','APP_PKG_LIST') pkgUrl = Slider.getAppPackageBaseUrl( isRU) + "/slider-app-packages/" + pkg_list util.downloadUrl(pkgUrl, path) with open(path, 'r') as f: for line in f: if line.startswith("hbase_pkg_url="): url = line.strip()[14:] break zipFile = os.path.join( os.getcwd(), "slider-hbase-app-package-%s.zip" % HBASE_VER_BUILD) logger.info("downloading " + url) util.downloadUrl(url, zipFile) return zipFile
def setupAcidDataset(testsuite, LOCAL_DIR): ddl_location = None if testsuite == 'acid': ddl_location = os.path.join(LOCAL_DIR, "ddl", "acid-tpch-tablesetup.sql") elif testsuite == 'unbucketed': ddl_location = os.path.join(LOCAL_DIR, "ddl", "acid-tpch-unbucketed-tablesetup.sql") else: assert 1 == 0, "The testsuite passed in not correct. Please use value 'acid' or 'unbuckted'" # change timezone on test machines Machine.resetTimeZoneOnCluster() # Download TPCH acids data tpch_newdata_dir = os.path.join(LOCAL_DIR, "tpch_newdata_5G") TPCH_STAGE_TGZ = os.path.join(LOCAL_DIR, "tpch_newdata_5G.tgz") if not os.path.isfile(TPCH_STAGE_TGZ): assert util.downloadUrl(Config.get('hive', 'TPCH_NEWDATA_5G_DNLD_URL'), TPCH_STAGE_TGZ) Machine.tarExtractAll(TPCH_STAGE_TGZ, LOCAL_DIR) # Load the acid tables in Hive HADOOPQA_USER = Config.get("hadoop", 'HADOOPQA_USER') HDFS.createDirectory("/tmp/lineitem_acid", user=HADOOPQA_USER, perm='777', force=True) HDFS.copyFromLocal(os.path.join(tpch_newdata_dir, "lineitem*"), "/tmp/lineitem_acid", HADOOPQA_USER) HDFS.chmod(None, 777, "/tmp/lineitem_acid", recursive=True) exit_code, stdout, stderr = Hive.runQueryOnBeeline( ddl_location, hivevar={'HDFS_LOCATION': '/tmp'}, logoutput=True, queryIsFile=True ) assert exit_code == 0, "Failed to populate the TPCH acid data in Hive"
def setupHS2ConcurrencyDataset(): logger.info("Setup test data") data_dir = os.path.join(Config.getEnv('ARTIFACTS_DIR'), "hs2concur-test-data") data_tgz = os.path.join(Config.getEnv('WORKSPACE'), "hs2concur-test-data.tgz") if not os.path.isfile(data_tgz): assert util.downloadUrl(Config.get('hive', 'HS2CONCURR_TEST_DATA'), data_tgz) Machine.tarExtractAll(data_tgz, data_dir) # load data into HDFS hdfs_user = Config.get("hadoop", 'HDFS_USER') HDFS.createDirectory("/tmp/hs2data", user=hdfs_user, perm='777', force=True) HDFS.createDirectory("/tmp/hs2data/student", perm='777', force=True) HDFS.copyFromLocal(os.path.join(data_dir, 'studenttab10k'), "/tmp/hs2data/student") HDFS.createDirectory("/tmp/hs2data/voter", perm='777', force=True) HDFS.copyFromLocal(os.path.join(data_dir, 'votertab10k'), "/tmp/hs2data/voter") query = """drop table if exists student_txt; create external table student_txt (name string, age int, gpa double) row format delimited fields terminated by '\\t' stored as textfile location '/tmp/hs2data/student'; drop table if exists voter_txt; create external table voter_txt (name string, age int, registration string, contributions float) row format delimited fields terminated by '\\t' stored as textfile location '/tmp/hs2data/voter'; drop table if exists student; create table student (name string, age int, gpa double) CLUSTERED BY (name) INTO 20 BUCKETS STORED AS ORC TBLPROPERTIES('transactional'='true'); drop table if exists voter; create table voter (name string, age int, registration string, contributions float) CLUSTERED BY (name) INTO 20 BUCKETS STORED AS ORC TBLPROPERTIES('transactional'='true'); Insert into table student select * from student_txt; Insert into table voter select * from voter_txt;""" exit_code, stdout, stderr = Hive.runQueryOnBeeline(query, readFromFile=True, logoutput=True) assert exit_code == 0, "Test data creation failed"
def setupMondrianDataset(): DATABASE_NAME = 'foodmart' LOCAL_DATA_DIR = os.path.join(Config.getEnv('ARTIFACTS_DIR'), DATABASE_NAME) FOODMART_DDL = os.path.join(LOCAL_DATA_DIR, "foodmart.ddl") HADOOPQA_USER = Config.get("hadoop", 'HADOOPQA_USER') logger.info("Setup Mondrian dataset") if not os.path.exists(LOCAL_DATA_DIR): MONDRIAN_DATA_TGZ = LOCAL_DATA_DIR + ".tgz" assert util.downloadUrl(Config.get('hive', 'MONDRIAN_DATASET'), MONDRIAN_DATA_TGZ) Machine.tarExtractAll(MONDRIAN_DATA_TGZ, Config.getEnv('ARTIFACTS_DIR')) assert os.path.isdir(LOCAL_DATA_DIR) logger.info("create foodmart database and tables") HDFS.createDirectory("/tmp/mondrian", HADOOPQA_USER, perm='777', force=True) HDFS.copyFromLocal(LOCAL_DATA_DIR, "/tmp/mondrian", HADOOPQA_USER) HDFS.chmod(None, 777, "/tmp/mondrian", recursive=True) exit_code, stdout, stderr = Hive.runQueryOnBeeline( FOODMART_DDL, hivevar={ 'DB': 'foodmart', 'LOCATION': '/tmp/mondrian/foodmart' }, logoutput=True, queryIsFile=True ) assert exit_code == 0, "Unable to deploy foodmart dataset"
def setupMergeScaleDataset(LOCAL_DIR): # change timezone on test machines Machine.resetTimeZoneOnCluster() # Download the TPCH dataset if not there tpch_data_dir = os.path.join(LOCAL_DIR, "data") TPCH_DATA_TGZ = os.path.join(LOCAL_DIR, "tpch_data.tgz") if not os.path.isfile(TPCH_DATA_TGZ): assert util.downloadUrl(Config.get('hive', 'TPCH_DNLD_URL'), TPCH_DATA_TGZ) Machine.tarExtractAll(TPCH_DATA_TGZ, LOCAL_DIR) # Load the tables in Hive HADOOPQA_USER = Config.get("hadoop", 'HADOOPQA_USER') HDFS.createDirectory("/tmp/tpch", user=HADOOPQA_USER, perm='777', force=True) HDFS.copyFromLocal(tpch_data_dir, "/tmp/tpch", user=HADOOPQA_USER) HDFS.chmod(None, 777, "/tmp/tpch", recursive=True) exit_code, stdout, stderr = Hive.runQueryOnBeeline( os.path.join(LOCAL_DIR, "ddl", "merge-tpch-tablesetup.sql"), hivevar={'HDFS_LOCATION': '/tmp/tpch/data'}, logoutput=True, queryIsFile=True ) assert exit_code == 0, "Failed to populate the TPCH data in Hive" # Download TPCH staging data tpch_stage_dir = os.path.join(LOCAL_DIR, "tpch_newdata_5G") TPCH_STAGE_TGZ = os.path.join(LOCAL_DIR, "tpch_newdata_5G.tgz") if not os.path.isfile(TPCH_STAGE_TGZ): assert util.downloadUrl(Config.get('hive', 'TPCH_NEWDATA_5G_DNLD_URL'), TPCH_STAGE_TGZ) Machine.tarExtractAll(TPCH_STAGE_TGZ, LOCAL_DIR) # Load the staged tables in Hive HDFS.createDirectory( "/tmp/lineitem_stage /tmp/orders_stage /tmp/delete_stage", user=HADOOPQA_USER, perm='777', force=True ) HDFS.copyFromLocal(os.path.join(tpch_stage_dir, "lineitem*"), "/tmp/lineitem_stage", HADOOPQA_USER) HDFS.copyFromLocal(os.path.join(tpch_stage_dir, "order*"), "/tmp/orders_stage", HADOOPQA_USER) HDFS.copyFromLocal(os.path.join(tpch_stage_dir, "delete*"), "/tmp/delete_stage", HADOOPQA_USER) HDFS.chmod(None, 777, "/tmp/lineitem_stage /tmp/orders_stage /tmp/delete_stage", recursive=True) exit_code, stdout, stderr = Hive.runQueryOnBeeline( os.path.join(LOCAL_DIR, "ddl", "merge-staged-tpch-tablesetup.sql"), hivevar={'HDFS_LOCATION': '/tmp'}, logoutput=True, queryIsFile=True ) assert exit_code == 0, "Failed to populate the TPCH staging data in Hive"
def setupElasticSearch(cls): # if already downloaded, skip if os.path.isdir(cls._es_home): return # download and install if Machine.type() == "Linux": tarballpath = os.path.join( cls._toolsdir, "elasticsearch-%s.tar.gz" % cls._es_version) if not os.path.isfile(tarballpath): assert util.downloadUrl(cls._downloadurlunix, tarballpath) Machine.tarExtractAll(tarballpath, cls._toolsdir, mode='r:gz') else: zippath = os.path.join(cls._toolsdir, "elasticsearch-%s.zip" % cls._es_version) if not os.path.isfile(zippath): assert util.downloadUrl(cls._downloadurlwin, zippath) Machine.unzipExtractAll(zippath, cls._toolsdir) assert os.path.exists(cls._es_home)
def setupActiveMQ(cls): # if already downloaded, skip if os.path.isdir(cls._activemq_home): return # download and install if Machine.type() == "Linux": tarballpath = os.path.join(cls._toolsdir, "apache-activemq-5.8.0-bin.tar.gz") if not os.path.isfile(tarballpath): assert util.downloadUrl(cls._downloadurlunix, tarballpath) Machine.tarExtractAll(tarballpath, cls._toolsdir, mode='r:gz') else: zippath = os.path.join(cls._toolsdir, "apache-activemq-5.8.0-bin.zip") if not os.path.isfile(zippath): assert util.downloadUrl(cls._downloadurlwin, zippath) Machine.unzipExtractAll(zippath, cls._toolsdir) assert os.path.exists(cls._activemq_home)
def setupHS2ConcurrTestData(stdauth=True): # hive.support.concurrency is not in the whitelist, as this is a server setting and not something that user should/can set in a session. # In a case of Ranger and SQL std authorization, set hive.support.concurrency to true and restart HS2 changes = { 'hive-site.xml': { 'hive.txn.manager': 'org.apache.hadoop.hive.ql.lockmgr.DbTxnManager', 'hive.support.concurrency': 'true', 'hive.compactor.initiator.on': 'true', 'hive.compactor.worker.threads': '3', 'hive.compactor.check.interval': '10', 'hive.timedout.txn.reaper.interval': '20s' }, 'hiveserver2-site.xml': { 'hive.compactor.initiator.on': 'false', 'hive.exec.dynamic.partition.mode': 'nonstrict' } } if not Hive.isHive2(): changes['hiveserver2-site.xml']['hive.enforce.bucketing'] = 'true' else: changes['hiveserver2-site.xml']['hive.server2.enable.doAs'] = 'false' changes['hiveserver2-site.xml']['hive.txn.manager'] = 'org.apache.hadoop.hive.ql.lockmgr.DbTxnManager' changes['hiveserver2-site.xml']['hive.support.concurrency'] = 'true' Hive.modifyConfig(changes) time.sleep(60) data_dir = os.path.join(Config.getEnv('ARTIFACTS_DIR'), "hs2concur-test-data") data_tgz = os.path.join(Config.getEnv('WORKSPACE'), "hs2concur-test-data.tgz") if not os.path.isfile(data_tgz): assert util.downloadUrl(Config.get('hive', 'HS2CONCURR_TEST_DATA'), data_tgz) Machine.tarExtractAll(data_tgz, data_dir) # load data into HDFS hdfs_user = Config.get("hadoop", 'HDFS_USER') test_user = Config.get("hadoop", 'HADOOPQA_USER') HDFS.createDirectory("/tmp/hs2data", user=test_user, perm='777', force=True) HDFS.createDirectory("/tmp/hs2data/student", user=test_user, perm='777', force=True) HDFS.copyFromLocal(os.path.join(data_dir, 'studenttab10k'), "/tmp/hs2data/student") HDFS.createDirectory("/tmp/hs2data/voter", perm='777', force=True) HDFS.copyFromLocal(os.path.join(data_dir, 'votertab10k'), "/tmp/hs2data/voter") HDFS.createDirectory("/tmp/hs2data/customer_address", perm='777', force=True) HDFS.copyFromLocal(os.path.join(data_dir, 'customer_address10k'), "/tmp/hs2data/customer_address") query = """drop table if exists student; create external table student (name string, age int, gpa double) row format delimited fields terminated by '\\t' stored as textfile location '/tmp/hs2data/student'; drop table if exists voter; create external table voter (name string, age int, registration string, contributions float) row format delimited fields terminated by '\\t' stored as textfile location '/tmp/hs2data/voter'; drop table if exists customer_address; create external table customer_address (ca_address_sk int, ca_address_id string, ca_street_number string, ca_street_name string, ca_street_type string, ca_suite_number string, ca_city string, ca_county string, ca_state string, ca_zip string, ca_country string, ca_gmt_offset decimal(5,2), ca_location_type string) row format delimited fields terminated by '|' stored as textfile location '/tmp/hs2data/customer_address'; drop table if exists customer_address_partitioned; create table customer_address_partitioned (ca_address_sk int, ca_address_id string, ca_street_number string, ca_street_name string, ca_street_type string, ca_suite_number string, ca_city string, ca_county string, ca_state string, ca_zip string, ca_country string, ca_gmt_offset decimal(5,2)) partitioned by (ca_location_type string) clustered by (ca_state) into 50 buckets stored as orc tblproperties('transactional'='true'); insert into table customer_address_partitioned partition(ca_location_type) select ca_address_sk, ca_address_id, ca_street_number, ca_street_name, ca_street_type, ca_suite_number, ca_city, ca_county, ca_state, ca_zip, ca_country, ca_gmt_offset, ca_location_type from customer_address;""" if stdauth: query += "\ngrant SELECT, INSERT, UPDATE, DELETE on table student to role public with grant option;" query += "\ngrant SELECT, INSERT, UPDATE, DELETE on table voter to role public with grant option;" query += "\ngrant SELECT, INSERT, UPDATE, DELETE on table customer_address_partitioned to role public with grant option;" exit_code, stdout, stderr = Hive.runQueryOnBeeline(query, readFromFile=True, logoutput=True) assert exit_code == 0, "Test data creation failed"
def setupSolr(cls): # if already downloaded, skip if os.path.isdir(cls._solr_home): return # download and install tarballpath = os.path.join(cls._toolsdir, "solr-%s.tgz" % cls._solr_version) if not os.path.isfile(tarballpath): assert util.downloadUrl(cls._downloadurl, tarballpath) Machine.tarExtractAll(tarballpath, cls._toolsdir, mode='r:gz')
def getZipFileAndBuild(cls, buildNo=""): """ This method is deprecated and used in ruSlider.py only DO NOT USE this method. """ zipFile = "" path = os.path.join(Config.getEnv('ARTIFACTS_DIR'), "pkg-list_qe.txt") #pkgUrl = Config.get('slider','APP_PKG_LIST') pkgUrl = Slider.getAppPackageBaseUrl( ) + "/slider-app-packages/pkg-list_qe.txt" if buildNo != "": pkgUrl = "http://s3.amazonaws.com/dev.hortonworks.com/HDP/" + Slider.getAppPackageOS( ) + "/2.x/BUILDS/" + buildNo + "/slider-app-packages/pkg-list_bn.txt" util.downloadUrl(pkgUrl, path) f = open(path, "r") buildNo = 0 for line in f: if line.startswith("hbase_pkg_url="): line = line.strip() url = line[14:] # hbase_ver has the format like 0.98.4.2.2.1.0 # we are not using cls.HBASE_VER here because in rollung-upgrade # the buildNo are different for current and base build get = re.search("\d\.\d{2}\.\d\.\d\.\d\.\d.\d", url) hbase_ver = get.group(0) index = line.find(hbase_ver) # skip version index = index + 15 buildNo = line[index:-12] logger.info("build# is " + buildNo) zipFile = os.path.join( os.getcwd(), "slider-hbase-app-package-" + hbase_ver + "-" + buildNo + "-hadoop2.zip") logger.info("downloading " + url) util.downloadUrl(url, zipFile) f.close() return buildNo, zipFile
def setupAnt(cls): # if dir exists return as its already setup if os.path.isdir(cls._ant_home): return # check if tarball exists or not before downloading it tarName = "apache-ant-" + cls._version + ".tar.gz" tarballPath = os.path.join(cls._worksapce, tarName) if not os.path.isfile(tarballPath): # download ant assert util.downloadUrl(cls._ant_download_url, tarballPath) # now untar ant on windows you have to run the tar cmd from the dir where # the file exists else it fails Machine.tarExtractAll(filepath=tarballPath, outpath=cls._tools_path, mode='r:gz') assert os.path.isfile(cls._ant_cmd)
def setupTableauDataset(): LOCAL_DATA_DIR = os.path.join(Config.getEnv('ARTIFACTS_DIR'), "tableau") DATA_DIR = os.path.join(LOCAL_DATA_DIR, 'data') SCHEMA_SQL_DIR = os.path.join(LOCAL_DATA_DIR, 'schema_3.0') HIVE_TABLES = [ 'Batters', 'Calcs', 'DateBins', 'DateTime', 'Election', 'FischerIris', 'Loan', 'NumericBins', 'REI', 'SeattleCrime', 'Securities', 'SpecialData', 'Staples', 'Starbucks', 'UTStarcom', 'xy' ] TABLEAU_TEST_DIR = "/user/hrt_qa/tableau" DATABASE_NAME = 'tableau' logger.info("Setup Tableau dataset") if not os.path.exists(LOCAL_DATA_DIR): TABLEAU_DATA_TGZ = LOCAL_DATA_DIR + ".tgz" assert util.downloadUrl(Config.get('hive', 'TABLEAU_DATASET'), TABLEAU_DATA_TGZ) Machine.tarExtractAll(TABLEAU_DATA_TGZ, Config.getEnv('ARTIFACTS_DIR')) assert os.path.isdir(LOCAL_DATA_DIR) logger.info("create test directory on hdfs to store tableau data files") HDFS.createDirectory(TABLEAU_TEST_DIR, user=HDFS_USER, perm='777', force=True) logger.info("create tableau database before creating tables") Hive.runQueryOnBeeline("DROP DATABASE IF EXISTS %s" % DATABASE_NAME) Hive.runQueryOnBeeline("CREATE DATABASE IF NOT EXISTS %s" % DATABASE_NAME) for tbl in HIVE_TABLES: hdfsDir = TABLEAU_TEST_DIR + '/%s' % tbl hdfsFile = hdfsDir + '/%s' % tbl localFile = os.path.join(DATA_DIR, '%s.tbl' % tbl) sqlFile = os.path.join(SCHEMA_SQL_DIR, '%s.sql' % tbl) logger.info("create directory for %s table" % tbl) exit_code, stdout = HDFS.createDirectory(hdfsDir, perm='777', force=True) assert exit_code == 0, 'Could not create dir for table %s on hdfs.' % tbl logger.info("copy file for table %s to hdfs" % tbl) exit_code, stdout = HDFS.copyFromLocal(localFile, hdfsFile) assert exit_code == 0, 'Could not copy file for table %s to hdfs.' % tbl logger.info("create %s table " % tbl) # thing-to-do Modify Hive.runQueryonBeeline to accept query file name exit_code, stdout, stderr = Hive.runQueryOnBeeline( ReadFromFile(sqlFile), readFromFile=True, hivevar={'HDFS_LOCATION': hdfsDir}, logoutput=True ) assert exit_code == 0, '%s table creation failed' % tbl
def setupJava7(cls): # if dir exists return as its already setup # check if tarball exists or not before downloading it tarName = "jdk-7u80-linux-x64.tar.gz" jdk7_folder_name = "jdk1.7.0_80" jdk7_java_home = os.path.join(cls._tools_path, jdk7_folder_name) if os.path.isdir(jdk7_java_home): logger.info("JDK 7 already installed, skipping setup of JDK 7") return jdk7_java_home tarballPath = os.path.join(cls._tools_path, tarName) if not os.path.isfile(tarballPath): # download java logger.info("JDK 7 downloading") assert util.downloadUrl(cls._jdk_download_url_7, tarballPath) # now install java Machine.run('chmod 755 ' + tarballPath) Machine.run('cd ' + cls._tools_path + '; tar zxvf ' + tarballPath) return jdk7_java_home
def setupJava(cls): # if dir exists return as its already setup if os.path.isdir(cls._java_home): logger.info("JDK already installed, skipping setup of JDK") return cls._java_home # check if tarball exists or not before downloading it tarName = "jdk-8u112-linux-x64.tar.gz" tarballPath = os.path.join(cls._tools_path, tarName) if not os.path.isfile(tarballPath): # download java assert util.downloadUrl(cls._jdk_download_url, tarballPath) # now install java Machine.run('chmod 755 ' + tarballPath) Machine.run('cd ' + cls._tools_path + '; tar zxvf ' + tarballPath) #Machine.run('echo A | .' + tarballPath + ' -noregister 2>&1') #Machine.tarExtractAll(filepath=tarballPath, outpath = cls._tools_path, mode='r:gz') print cls._java_home assert os.path.isfile(cls._java_cmd) return cls._java_home
def downloadDataset(dataDir, dataTgz, downloadUrl, hdfsLocalCopy, textDataDir): HDFS.createDirectory(HCAT_TEST_DIR, user=HDFS_USER, perm='777', force=True) HDFS.createDirectory(HDFS_TEST_DIR, user=HDFS_USER, perm='777', force=True) # change timezone on test machines Machine.resetTimeZoneOnCluster() # Download the TPCDS dataset if not there if not os.path.isfile(dataTgz): assert util.downloadUrl(downloadUrl, dataTgz) Machine.tarExtractAll(dataTgz, dataDir) os.makedirs(hdfsLocalCopy) for filename in os.listdir(textDataDir): hdfs_localcopy_table_dir = os.path.join(hdfsLocalCopy, filename[:-4]) os.mkdir(hdfs_localcopy_table_dir) shutil.copy(os.path.join(textDataDir, filename), hdfs_localcopy_table_dir) HDFS.copyFromLocal(hdfsLocalCopy, HDFS_TEST_DIR) HDFS.chmod(None, '777', HDFS_TEST_DIR, recursive=True)
def setupTestData(stdauth=True): data_dir = os.path.join(Config.getEnv('ARTIFACTS_DIR'), "hive-test-data") data_tgz = os.path.join(Config.getEnv('WORKSPACE'), "hive-simple-test-data.tgz") if not os.path.isfile(data_tgz): assert util.downloadUrl(Config.get('hive', 'HIVE_TEST_DATA'), data_tgz) Machine.tarExtractAll(data_tgz, data_dir) # load data into HDFS HDFS.createDirectory("/tmp/hs2data", user=HDFS_USER, perm='777', force=True) HDFS.createDirectory("/tmp/hs2data/student", perm='777', force=True) HDFS.copyFromLocal(os.path.join(data_dir, 'studenttab10k'), "/tmp/hs2data/student") HDFS.createDirectory("/tmp/hs2data/voter", perm='777', force=True) HDFS.copyFromLocal(os.path.join(data_dir, 'votertab10k'), "/tmp/hs2data/voter") query = """drop table if exists student; create external table student (name string, age int, gpa double) row format delimited fields terminated by '\\t' stored as textfile location '/tmp/hs2data/student'; drop table if exists voter; create external table voter (name string, age int, registration string, contributions float) row format delimited fields terminated by '\\t' stored as textfile location '/tmp/hs2data/voter';""" if stdauth: query += "\ngrant SELECT, INSERT, UPDATE, DELETE on table student to role public with grant option;" query += "\ngrant SELECT, INSERT, UPDATE, DELETE on table voter to role public with grant option;" exit_code, stdout, stderr = Hive.runQueryOnBeeline(query, readFromFile=True, logoutput=True) assert exit_code == 0, "Test data creation failed"
def setupMaven(cls, version=_version, mvn_url=_maven_download_url): # if dir exists return as its already setup mvn_home = os.path.join(cls._tools_path, 'apache-maven-' + version) if os.path.isdir(mvn_home): return #if os.path.isdir(cls._maven_home): # return # check if tarball exists or not before downloading it tarName = "apache-maven-" + version + ".tar.gz" #tarName = "apache-maven-" +cls._version + ".tar.gz" tarballPath = os.path.join(cls._worksapce, tarName) if not os.path.isfile(tarballPath): # download maven assert util.downloadUrl(mvn_url, tarballPath) #assert util.downloadUrl(cls._maven_download_url, tarballPath) # now extract maven Machine.tarExtractAll(filepath=tarballPath, outpath=cls._tools_path, mode='r:gz') mvn_cmd = os.path.join(mvn_home, 'bin', 'mvn') assert os.path.isfile(mvn_cmd)