def s3_path(self): """get or create a location on S3 for the serde jar""" if 'CSV_SERDE_JAR_S3' in os.environ: # known location on S3 serde = os.environ['CSV_SERDE_JAR_S3'] else: if self._s3_base_path is None: raise ValueError("must specify the S3 scratch URI") # ensure the jar is up on S3 jar_path = self._s3_base_path + 'jars/csv-serde.jar' upload_file_to_s3(self.jar, jar_path) os.environ['CSV_SERDE_JAR_S3'] = serde = jar_path return serde
def s3_path(self): """get or create a location on S3 for the serde jar""" if "CSV_SERDE_JAR_S3" in os.environ: # known location on S3 serde = os.environ["CSV_SERDE_JAR_S3"] else: if self._s3_base_path is None: raise ValueError("must specify the S3 scratch URI") # ensure the jar is up on S3 jar_path = self._s3_base_path + "jars/csv-serde.jar" upload_file_to_s3(self.jar, jar_path) os.environ["CSV_SERDE_JAR_S3"] = serde = jar_path return serde
def _generate_and_upload_hive_script(self): self._generate_hive_script(self.data_path) upload_file_to_s3(self.local_script_file, self.script_path)