def download(url, target=None, force=False): """ Download and storage file from a url """ if not is_url(url): raise TypeError("Str '%s' is not a valid url." % url) storage_folder = target or settings.get('STORAGE_TEMPLATES') auth = str(settings.get('REPOSITORY_AUTH')).replace("'", '') file_name = path.basename(urlsplit(url).path) file_path = storage_folder + file_name if force or not path.isfile(file_path): Storage.create_folders(storage_folder) try: request = Request(url) if auth: request.add_header('Authorization', auth) with urlopen(request) as response: Storage.file(file_path, response.read(), 'wb') logger.log(url, 'download DONE!') except HTTPError as e: e.msg += ": URL '%s' cannot be downloaded" % url raise e else: logger.log(url, 'from CACHE!') return file_path
def __init__(self, name): sep = ':' if sep not in name: raise AttributeError( "Attr 'name' has malformed value. It must have ':' splitting path and version") self.local_path = None self.repository_url = settings.get('REPOSITORY') self.repository_type = settings.get('REPOSITORY_TYPE') self.repository_custom = settings.get('REPOSITORY_CUSTOM_PATTERN') self.path, self.version = name.split(sep) self._check_settings()
def generate(key_pass): """ Generate secret key from key pass """ b_key_pass = key_pass.encode(settings.get('ENCODING')) sha256 = SHA256.new(b_key_pass) secret_key = sha256.digest() secret_store = binascii.hexlify(secret_key).decode( settings.get('ENCODING')) Storage.file(settings.get('STORAGE_KEY_PATH'), secret_store) logger.log("Generated secret key '{0}' " "and saved at '{1}'".format( secret_store, settings.get('STORAGE_KEY_PATH'))) return secret_store
def dict_from_ini(self): """ Load instruction from a recipe with ini format """ self.parser = configparser.ConfigParser() if self.case_sensitive: self.parser.optionxform = str self.parser.read(self.recipe_file, encoding=settings.get('ENCODING')) if self.parser.sections(): curr_template = None for section in self.parser.sections(): name = section.rsplit(':', 1)[0] if name != curr_template: curr_template = name variables = self._get_values(self.parser, name + ':variables') secrets = self._get_values(self.parser, name + ':secrets') template = self._get_values(self.parser, name + ':template') if template: template['name'] = name else: raise AttributeError('Attribute template is required') self.instructions.append( Instruction(template, variables, secrets)) else: raise FileExistsError('Unable to read instructions from file')
def list(all_info=False): """ List of recipes saved in index """ recipes = Storage.json(settings.get('STORAGE_RECIPE_INDEX')) meta = _IndexRecipe.calc_length(recipes) meta['id'] = 64 if all_info else 9 extra_space = 8 list_items = '' for key in recipes.keys(): recipe = recipes[key] recipe_id = key[:meta['id']] created = recipe['datetime'] if all_info else recipe['datetime'][:19] list_items += recipe_id + (' ' * (meta['id'] + extra_space - len(recipe_id))) for attr_name in ['remote', 'version', 'filename']: list_items += (recipe[attr_name] + (' ' * (meta[attr_name] + extra_space - len(recipe[attr_name])))) list_items += created + '\n' header = ListRecipes._list_header(meta, extra_space) logger.log(header + list_items)
def _generate_id(self): """ Generate a hash as id to a recipe based on path and version """ str_base = self.remote + self.version str_hash = hashlib.sha256(str_base.encode(settings.get('ENCODING'))) return str_hash.hexdigest()
def indexing(self, update=False): """ Indexing recipes locally """ if not self.is_indexed() or update: self.index[self.id] = {'remote': self.remote, 'version': self.version, 'filename': self.filename, 'datetime': str(datetime.now())} Storage.json(settings.get('STORAGE_RECIPE_INDEX'), self.index)
def encrypt(self, raw): """ Encrypt and return an hexadecimal utf-8 as default format """ b_raw = raw.encode(settings.get('ENCODING')) cipher = AES.new(self.key, AES.MODE_EAX) b_cipher, tag = cipher.encrypt_and_digest(b_raw) return self._build_encrypt(cipher.nonce, b_cipher, tag)
def pull(self, force): """ Pull recipe by path and version """ url = self._format_url() filename = url.rsplit('/', 1)[1] index = _IndexRecipe(self.path, self.version, filename) target = settings.get('STORAGE_RECIPE') + index.id + '/' self.local_path = download(url, target, force) index.indexing(update=force)
def replace(self, mapping): """ Replace variable based on mapping key """ try: if settings.get('RECIPE_CASE_SENSITIVE'): return super(BakerTemplate, self).substitute(mapping) else: return self.ignore_case_substitute(mapping) except KeyError as e: raise KeyError('Missing variable %s' % e)
def decrypt(self, encrypt): """ Decrypt values from a hexadecimal utf-8 as default format to plaintext """ nonce, tag, b_cipher = self._split_encrypt(encrypt) cipher = AES.new(self.key, AES.MODE_EAX, nonce=nonce) try: return cipher.decrypt_and_verify(b_cipher, tag).decode( settings.get('ENCODING')) except ValueError: raise ValueError("Encryption '%s' is corrupted." % encrypt)
def init(): """ Initialize logger for all application """ global LOGGER LOGGER = logging.getLogger() level = 'DEBUG' if settings.get('DEBUG') else 'INFO' handler = logging.StreamHandler(sys.stdout) handler.setFormatter(logging.Formatter('%(message)s')) LOGGER.addHandler(handler) LOGGER.setLevel(level)
def __init__(self, file, case_sensitive=False): self.parser = None self.instructions = [] self.case_sensitive = case_sensitive or settings.get( 'RECIPE_CASE_SENSITIVE') self.recipe_file = file filename = file.lower() if filename.endswith('.cfg'): self.dict_from_ini() # elif filename.endswith('.yml'): # TODO: Add support recipes via yaml file # self.dict_from_yaml() else: raise FileExistsError('Unsupported file format')
def replace(self): """ Replace variables in template file based on recipe instructions """ for instruction in self.instructions: target = instruction.template template_path = instruction.template replaced = Storage.file(template_path) if instruction.variables: template = BakerTemplate(replaced) replaced = template.replace(instruction.variables) if hasattr(instruction, 'path'): target = instruction.path if settings.get('TEMPLATE_EXT') and target.endswith( settings.get('TEMPLATE_EXT')): ext_size = len(settings.get('TEMPLATE_EXT')) + 1 target = target[:-ext_size] Storage.file(target, content=replaced) self._add_file_permission(instruction, target) logger.log(instruction.name, instruction.template, target)
def remove(rid): """ Remove locally recipe by id """ location = settings.get('STORAGE_RECIPE_INDEX') index = Storage.json(location) if len(rid) != 64: found = list(filter(lambda idx: idx[:9] == rid, index)) if found: rid = found[0] del index[rid] Storage.json(location, index) logger.log("Removed recipe '%s'" % rid)
def download(url, target=None, force=False): """ Download and storage file from a url """ if not is_url(url): raise TypeError("Str '%s' is not a valid url." % url) storage_folder = target or settings.get('STORAGE_TEMPLATES') file_name = path.basename(urlsplit(url).path) file_path = storage_folder + file_name if force or not path.isfile(file_path): Storage.create_folders(storage_folder) try: urlretrieve(url, file_path) logger.log(url, 'download DONE!') except HTTPError as e: e.msg += ": URL '%s' cannot be downloaded" % url raise e else: logger.log(url, 'from CACHE!') return file_path
def __init__(self, remote, version, filename): self.remote = remote self.version = version self.filename = filename self.id = self._generate_id() self.index = Storage.json(settings.get('STORAGE_RECIPE_INDEX'))
def key(self): """ Read secret key from storage file """ return Storage.file(settings.get('STORAGE_KEY_PATH'))
def _to_bin(hexadecimal): """ Convert hexadecimal string encoded in utf-8 as default into binary """ return binascii.unhexlify(hexadecimal.encode(settings.get('ENCODING')))
def _to_hex(binary): """ Convert binary string into hexadecimal decoded in utf-8 as default """ return binascii.hexlify(binary).decode(settings.get('ENCODING'))