def __init__(self, content, dir=None, name='LOGGING_CONFIG'): self.content = content self.settings_root = tempfile.mkdtemp() filename = "{}.py".format(SETTINGS_DEFAULT_NAME) if dir: # Replace slashes by dots self.module = dir.replace('/', '.') + '.' + SETTINGS_DEFAULT_NAME + '.' + name # Create the directory structure dir_path = os.path.join(self.settings_root, dir) mkdir_p(dir_path) # Add the __init__ for the directories # This is required for Python 2.7 basedir = self.settings_root for part in dir.split('/'): open(os.path.join(basedir, '__init__.py'), 'w').close() basedir = os.path.join(basedir, part) open(os.path.join(basedir, '__init__.py'), 'w').close() self.settings_file = os.path.join(dir_path, filename) else: self.module = SETTINGS_DEFAULT_NAME + '.' + name self.settings_file = os.path.join(self.settings_root, filename)
def get_results(self, ti=None, fp=None, inline=True, delim=None, fetch=True): """ Get results (or just s3 locations) of a command from Qubole and save into a file :param ti: Task Instance of the dag, used to determine the Quboles command id :param fp: Optional file pointer, will create one and return if None passed :param inline: True to download actual results, False to get s3 locations only :param delim: Replaces the CTL-A chars with the given delim, defaults to ',' :param fetch: when inline is True, get results directly from s3 (if large) :return: file location containing actual results or s3 locations of results """ if fp is None: iso = datetime.datetime.utcnow().isoformat() logpath = os.path.expanduser(configuration.get('core', 'BASE_LOG_FOLDER')) resultpath = logpath + '/' + self.dag_id + '/' + self.task_id + '/results' configuration.mkdir_p(resultpath) fp = open(resultpath + '/' + iso, 'wb') if self.cmd is None: cmd_id = ti.xcom_pull(key="qbol_cmd_id", task_ids=self.task_id) self.cmd = self.cls.find(cmd_id) self.cmd.get_results(fp, inline, delim, fetch) fp.flush() fp.close() return fp.name