def download(url, target=None, force=False):
"""
Download and storage file from a url
"""
if not is_url(url):
raise TypeError("Str '%s' is not a valid url." % url)
storage_folder = target or settings.get('STORAGE_TEMPLATES')
auth = str(settings.get('REPOSITORY_AUTH')).replace("'", '')
file_name = path.basename(urlsplit(url).path)
file_path = storage_folder + file_name
if force or not path.isfile(file_path):
Storage.create_folders(storage_folder)
try:
request = Request(url)
if auth:
request.add_header('Authorization', auth)
with urlopen(request) as response:
Storage.file(file_path, response.read(), 'wb')
logger.log(url, 'download DONE!')
except HTTPError as e:
e.msg += ": URL '%s' cannot be downloaded" % url
raise e
else:
logger.log(url, 'from CACHE!')
return file_path
def __init__(self, name):
sep = ':'
if sep not in name:
raise AttributeError(
"Attr 'name' has malformed value. It must have ':' splitting path and version")
self.local_path = None
self.repository_url = settings.get('REPOSITORY')
self.repository_type = settings.get('REPOSITORY_TYPE')
self.repository_custom = settings.get('REPOSITORY_CUSTOM_PATTERN')
self.path, self.version = name.split(sep)
self._check_settings()
def generate(key_pass):
"""
Generate secret key from key pass
"""
b_key_pass = key_pass.encode(settings.get('ENCODING'))
sha256 = SHA256.new(b_key_pass)
secret_key = sha256.digest()
secret_store = binascii.hexlify(secret_key).decode(
settings.get('ENCODING'))
Storage.file(settings.get('STORAGE_KEY_PATH'), secret_store)
logger.log("Generated secret key '{0}' "
"and saved at '{1}'".format(
secret_store, settings.get('STORAGE_KEY_PATH')))
return secret_store
def dict_from_ini(self):
"""
Load instruction from a recipe with ini format
"""
self.parser = configparser.ConfigParser()
if self.case_sensitive:
self.parser.optionxform = str
self.parser.read(self.recipe_file, encoding=settings.get('ENCODING'))
if self.parser.sections():
curr_template = None
for section in self.parser.sections():
name = section.rsplit(':', 1)[0]
if name != curr_template:
curr_template = name
variables = self._get_values(self.parser,
name + ':variables')
secrets = self._get_values(self.parser, name + ':secrets')
template = self._get_values(self.parser,
name + ':template')
if template:
template['name'] = name
else:
raise AttributeError('Attribute template is required')
self.instructions.append(
Instruction(template, variables, secrets))
else:
raise FileExistsError('Unable to read instructions from file')
def list(all_info=False):
"""
List of recipes saved in index
"""
recipes = Storage.json(settings.get('STORAGE_RECIPE_INDEX'))
meta = _IndexRecipe.calc_length(recipes)
meta['id'] = 64 if all_info else 9
extra_space = 8
list_items = ''
for key in recipes.keys():
recipe = recipes[key]
recipe_id = key[:meta['id']]
created = recipe['datetime'] if all_info else recipe['datetime'][:19]
list_items += recipe_id + (' ' * (meta['id'] + extra_space - len(recipe_id)))
for attr_name in ['remote', 'version', 'filename']:
list_items += (recipe[attr_name] +
(' ' * (meta[attr_name] + extra_space - len(recipe[attr_name]))))
list_items += created + '\n'
header = ListRecipes._list_header(meta, extra_space)
logger.log(header + list_items)
def _generate_id(self):
"""
Generate a hash as id to a recipe based on path and version
"""
str_base = self.remote + self.version
str_hash = hashlib.sha256(str_base.encode(settings.get('ENCODING')))
return str_hash.hexdigest()
def indexing(self, update=False):
"""
Indexing recipes locally
"""
if not self.is_indexed() or update:
self.index[self.id] = {'remote': self.remote, 'version': self.version,
'filename': self.filename, 'datetime': str(datetime.now())}
Storage.json(settings.get('STORAGE_RECIPE_INDEX'), self.index)
def encrypt(self, raw):
"""
Encrypt and return an hexadecimal utf-8 as default format
"""
b_raw = raw.encode(settings.get('ENCODING'))
cipher = AES.new(self.key, AES.MODE_EAX)
b_cipher, tag = cipher.encrypt_and_digest(b_raw)
return self._build_encrypt(cipher.nonce, b_cipher, tag)
def pull(self, force):
"""
Pull recipe by path and version
"""
url = self._format_url()
filename = url.rsplit('/', 1)[1]
index = _IndexRecipe(self.path, self.version, filename)
target = settings.get('STORAGE_RECIPE') + index.id + '/'
self.local_path = download(url, target, force)
index.indexing(update=force)
def replace(self, mapping):
"""
Replace variable based on mapping key
"""
try:
if settings.get('RECIPE_CASE_SENSITIVE'):
return super(BakerTemplate, self).substitute(mapping)
else:
return self.ignore_case_substitute(mapping)
except KeyError as e:
raise KeyError('Missing variable %s' % e)
def decrypt(self, encrypt):
"""
Decrypt values from a hexadecimal utf-8 as default format to plaintext
"""
nonce, tag, b_cipher = self._split_encrypt(encrypt)
cipher = AES.new(self.key, AES.MODE_EAX, nonce=nonce)
try:
return cipher.decrypt_and_verify(b_cipher, tag).decode(
settings.get('ENCODING'))
except ValueError:
raise ValueError("Encryption '%s' is corrupted." % encrypt)
def init():
"""
Initialize logger for all application
"""
global LOGGER
LOGGER = logging.getLogger()
level = 'DEBUG' if settings.get('DEBUG') else 'INFO'
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(logging.Formatter('%(message)s'))
LOGGER.addHandler(handler)
LOGGER.setLevel(level)
def __init__(self, file, case_sensitive=False):
self.parser = None
self.instructions = []
self.case_sensitive = case_sensitive or settings.get(
'RECIPE_CASE_SENSITIVE')
self.recipe_file = file
filename = file.lower()
if filename.endswith('.cfg'):
self.dict_from_ini()
# elif filename.endswith('.yml'): # TODO: Add support recipes via yaml file
# self.dict_from_yaml()
else:
raise FileExistsError('Unsupported file format')
def replace(self):
"""
Replace variables in template file based on recipe instructions
"""
for instruction in self.instructions:
target = instruction.template
template_path = instruction.template
replaced = Storage.file(template_path)
if instruction.variables:
template = BakerTemplate(replaced)
replaced = template.replace(instruction.variables)
if hasattr(instruction, 'path'):
target = instruction.path
if settings.get('TEMPLATE_EXT') and target.endswith(
settings.get('TEMPLATE_EXT')):
ext_size = len(settings.get('TEMPLATE_EXT')) + 1
target = target[:-ext_size]
Storage.file(target, content=replaced)
self._add_file_permission(instruction, target)
logger.log(instruction.name, instruction.template, target)
def remove(rid):
"""
Remove locally recipe by id
"""
location = settings.get('STORAGE_RECIPE_INDEX')
index = Storage.json(location)
if len(rid) != 64:
found = list(filter(lambda idx: idx[:9] == rid, index))
if found:
rid = found[0]
del index[rid]
Storage.json(location, index)
logger.log("Removed recipe '%s'" % rid)
def download(url, target=None, force=False):
"""
Download and storage file from a url
"""
if not is_url(url):
raise TypeError("Str '%s' is not a valid url." % url)
storage_folder = target or settings.get('STORAGE_TEMPLATES')
file_name = path.basename(urlsplit(url).path)
file_path = storage_folder + file_name
if force or not path.isfile(file_path):
Storage.create_folders(storage_folder)
try:
urlretrieve(url, file_path)
logger.log(url, 'download DONE!')
except HTTPError as e:
e.msg += ": URL '%s' cannot be downloaded" % url
raise e
else:
logger.log(url, 'from CACHE!')
return file_path
def __init__(self, remote, version, filename):
self.remote = remote
self.version = version
self.filename = filename
self.id = self._generate_id()
self.index = Storage.json(settings.get('STORAGE_RECIPE_INDEX'))
def key(self):
"""
Read secret key from storage file
"""
return Storage.file(settings.get('STORAGE_KEY_PATH'))
def _to_bin(hexadecimal):
"""
Convert hexadecimal string encoded in utf-8 as default into binary
"""
return binascii.unhexlify(hexadecimal.encode(settings.get('ENCODING')))
def _to_hex(binary):
"""
Convert binary string into hexadecimal decoded in utf-8 as default
"""
return binascii.hexlify(binary).decode(settings.get('ENCODING'))
