Esempio n. 1
0
 def s3_path(self):
     """get or create a location on S3 for the serde jar"""
     if 'CSV_SERDE_JAR_S3' in os.environ:
         # known location on S3
         serde = os.environ['CSV_SERDE_JAR_S3']
     else:
         if self._s3_base_path is None:
             raise ValueError("must specify the S3 scratch URI")
         # ensure the jar is up on S3
         jar_path = self._s3_base_path + 'jars/csv-serde.jar'
         upload_file_to_s3(self.jar, jar_path)
         os.environ['CSV_SERDE_JAR_S3'] = serde = jar_path
     return serde
Esempio n. 2
0
 def s3_path(self):
     """get or create a location on S3 for the serde jar"""
     if "CSV_SERDE_JAR_S3" in os.environ:
         # known location on S3
         serde = os.environ["CSV_SERDE_JAR_S3"]
     else:
         if self._s3_base_path is None:
             raise ValueError("must specify the S3 scratch URI")
         # ensure the jar is up on S3
         jar_path = self._s3_base_path + "jars/csv-serde.jar"
         upload_file_to_s3(self.jar, jar_path)
         os.environ["CSV_SERDE_JAR_S3"] = serde = jar_path
     return serde
Esempio n. 3
0
 def _generate_and_upload_hive_script(self):
     self._generate_hive_script(self.data_path)
     upload_file_to_s3(self.local_script_file, self.script_path)
Esempio n. 4
0
 def _generate_and_upload_hive_script(self):
     self._generate_hive_script(self.data_path)
     upload_file_to_s3(self.local_script_file, self.script_path)