class KeyImporter(object): def __init__(self): self.db = Storage() def _look_for_keypack(self): files = os.listdir(os.getcwd()) if settings.KEYPACK_FILENAME not in files: return False return True def _load_keypack(self): json_data = None try: with io.open(settings.KEYPACK_FILENAME, 'r', encoding='utf-8') as f: data = f.read() except Exception as err: log.debug(err, exc_info=True) else: try: json_data = json.loads(data) except Exception as err: log.debug(err, exc_info=True) return json_data def start(self): found = self._look_for_keypack() if found is False: return False keypack = self._load_keypack() if keypack is None: return False self.db.save(settings.CONFIG_DB_KEY_KEYPACK, keypack) return True
class KeyHandler(object): """KeyHandler object is used to manage keys used for signing updates Kwargs: app (obj): Config object to get config values from """ def __init__(self): self.db = Storage() self.key_encoding = 'base64' data_dir = os.getcwd() self.data_dir = os.path.join(data_dir, settings.USER_DATA_FOLDER) self.deploy_dir = os.path.join(self.data_dir, 'deploy') # Name of the keypack to import. It should be placed # in the root of the repo self.keypack_filename = os.path.join(data_dir, settings.CONFIG_DATA_FOLDER, settings.KEYPACK_FILENAME) # The name of the gzipped version file in # the pyu-data/deploy directory self.version_file = os.path.join(self.deploy_dir, settings.VERSION_FILE_FILENAME) # The name of the gzipped key file in # the pyu-data/deploy directory self.key_file = os.path.join(self.deploy_dir, settings.KEY_FILE_FILENAME) def sign_update(self): """Signs version file with private key Proxy method for :meth:`_add_sig` """ # Loads private key # Loads version file to memory # Signs Version file # Writes version file back to disk self._add_sig() def _load_private_keys(self): # Loads private key log.debug('Loading private key') # Loading keypack data from .pyupdater/config.pyu keypack_data = self.db.load(settings.CONFIG_DB_KEY_KEYPACK) private_key = None if keypack_data is not None: try: private_key = keypack_data['repo']['app_private'] except KeyError: # We will exit in _add_sig if private_key is None pass return private_key def _add_sig(self): # Adding new signature to version file # Raw private key will need to be converted into # a signing key object private_key_raw = self._load_private_keys() if private_key_raw is None: log.error('Private Key not found. Please ' 'import a keypack & try again') return # Load update manifest update_data = self._load_update_data() # We don't want to verify the signature if 'signature' in update_data: log.debug('Removing signatures from version file') del update_data['signature'] # We create a signature from the string update_data_str = json.dumps(update_data, sort_keys=True) private_key_raw = private_key_raw.encode('utf-8') # Creating signing key object private_key = ed25519.SigningKey(private_key_raw, encoding=self.key_encoding) log.debug("Signing update data") # Signs update data with private key signature = private_key.sign(six.b(update_data_str), encoding=self.key_encoding).decode() log.debug('Sig: %s', signature) # Create new dict from json string update_data = json.loads(update_data_str) # Add signatures to update data update_data['signature'] = signature log.debug('Adding signature to update data') # Write updated version file to .pyupdater/config.pyu self._write_update_data(update_data) # Write gzipped key file self._write_key_file() def _write_update_data(self, data): # Save update data to repo database self.db.save(settings.CONFIG_DB_KEY_VERSION_META, data) log.debug('Saved version meta data') log.debug('Upload manifest: \n%s', data) # Gzip update date with gzip.open(self.version_file, 'wb') as f: new_data = json.dumps(data) if six.PY2: f.write(new_data) else: f.write(bytes(new_data, 'utf-8')) log.debug('Created gzipped version manifest in deploy dir') def _write_key_file(self): keypack_data = self.db.load(settings.CONFIG_DB_KEY_KEYPACK) if keypack_data is None: log.error('Private Key not found. Please ' 'import a keypack & try again') return upload_data = keypack_data['upload'] with gzip.open(self.key_file, 'wb') as f: new_data = json.dumps(upload_data) if six.PY2: f.write(new_data) else: f.write(bytes(new_data, 'utf-8')) log.debug('Created gzipped key file in deploy dir') def _load_update_data(self): log.debug("Loading version data") update_data = self.db.load(settings.CONFIG_DB_KEY_VERSION_META) # If update_data is None, create a new one if update_data is None: update_data = {} log.error('Version meta data not found') self.db.save(settings.CONFIG_DB_KEY_VERSION_META, update_data) log.debug('Created new version meta data') log.debug('Version file loaded') return update_data
class ConfigManager(object): def __init__(self): self.cwd = os.getcwd() self.db = Storage() self.config_key = settings.CONFIG_DB_KEY_APP_CONFIG # Loads config from database (json file) def load_config(self): config_data = self.db.load(self.config_key) if config_data is None: config_data = {} config = Config() for k, v in config_data.items(): config[k] = v config.DATA_DIR = os.getcwd() return config def get_app_name(self): config = self.load_config() return config.APP_NAME # Saves config to database (json file) def save_config(self, obj): log.info('Saving Config') self.db.save(self.config_key, obj) log.info('Config saved') self.write_config_py(obj) log.info('Wrote client config') # Writes client config to client_config.py def write_config_py(self, obj): keypack_data = self.db.load(settings.CONFIG_DB_KEY_KEYPACK) if keypack_data is None: log.debug('*** Keypack data is None ***') public_key = None else: public_key = keypack_data['client']['offline_public'] filename = os.path.join(self.cwd, *obj.CLIENT_CONFIG_PATH) attr_str_format = " {} = '{}'\n" attr_format = " {} = {}\n" log.debug('Writing client_config.py') with open(filename, 'w') as f: f.write('class ClientConfig(object):\n') log.debug('Adding PUBLIC_KEY to client_config.py') f.write(attr_str_format.format('PUBLIC_KEY', public_key)) if hasattr(obj, 'APP_NAME'): log.debug('Adding APP_NAME to client_config.py') f.write(attr_str_format.format('APP_NAME', obj.APP_NAME)) if hasattr(obj, 'COMPANY_NAME'): log.debug('Adding COMPANY_NAME to client_config.py') f.write(attr_str_format.format('COMPANY_NAME', obj.COMPANY_NAME)) if hasattr(obj, 'UPDATE_URLS'): log.debug('Adding UPDATE_URLS to client_config.py') f.write(attr_format.format('UPDATE_URLS', obj.UPDATE_URLS)) if hasattr(obj, 'MAX_DOWNLOAD_RETRIES'): log.debug('Adding MAX_DOWNLOAD_RETRIES to client_config.py') f.write(attr_format.format('MAX_DOWNLOAD_RETRIES', obj.MAX_DOWNLOAD_RETRIES))
class Loader(object): """Loads & saves config file """ def __init__(self): self.cwd = os.getcwd() self.db = Storage() self.password = os.environ.get(settings.USER_PASS_ENV) self.config_key = settings.CONFIG_DB_KEY_APP_CONFIG def load_config(self): """Loads config from database (json file) Returns (obj): Config object """ config_data = self.db.load(self.config_key) if config_data is None: config_data = {} config = Config() for k, v in config_data.items(): config[k] = v config.DATA_DIR = os.getcwd() return config def get_app_name(self): config = self.load_config() return config.APP_NAME def save_config(self, obj): """Saves config file to pyupdater database Args: obj (obj): config object """ log.info('Saving Config') self.db.save(self.config_key, obj) log.info('Config saved') self._write_config_py(obj) log.info('Wrote client config') def _write_config_py(self, obj): """Writes client config to client_config.py Args: obj (obj): config object """ keypack_data = self.db.load(settings.CONFIG_DB_KEY_KEYPACK) if keypack_data is None: public_key = None else: public_key = keypack_data['client']['offline_public'] filename = os.path.join(self.cwd, *obj.CLIENT_CONFIG_PATH) attr_str_format = " {} = '{}'\n" attr_format = " {} = {}\n" with open(filename, 'w') as f: f.write('class ClientConfig(object):\n') if hasattr(obj, 'APP_NAME') and obj.APP_NAME is not None: f.write(attr_str_format.format('APP_NAME', obj.APP_NAME)) log.debug('Wrote APP_NAME to client config') if hasattr(obj, 'COMPANY_NAME') and obj.COMPANY_NAME is not None: f.write( attr_str_format.format('COMPANY_NAME', obj.COMPANY_NAME)) log.debug('Wrote COMPANY_NAME to client config') if hasattr(obj, 'UPDATE_URLS') and obj.UPDATE_URLS is not None: f.write(attr_format.format('UPDATE_URLS', obj.UPDATE_URLS)) log.debug('Wrote UPDATE_URLS to client config') f.write(attr_str_format.format('PUBLIC_KEY', public_key)) log.debug('Wrote PUBLIC_KEY to client config')
class KeyHandler(object): """KeyHanlder object is used to manage keys used for signing updates Kwargs: app (obj): Config object to get config values from """ def __init__(self): self.db = Storage() self.key_encoding = 'base64' data_dir = os.getcwd() self.data_dir = os.path.join(data_dir, settings.USER_DATA_FOLDER) self.deploy_dir = os.path.join(self.data_dir, 'deploy') self.keypack_filename = os.path.join(data_dir, settings.CONFIG_DATA_FOLDER, settings.KEYPACK_FILENAME) self.version_file = os.path.join(self.deploy_dir, settings.VERSION_FILE) self.key_file = os.path.join(self.deploy_dir, settings.KEY_FILE) def sign_update(self): """Signs version file with private key Proxy method for :meth:`_add_sig` """ # Loads private key # Loads version file to memory # Signs Version file # Writes version file back to disk self._add_sig() def _load_private_keys(self): # Loads private key log.debug('Loading private key') keypack_data = self.db.load(settings.CONFIG_DB_KEY_KEYPACK) private_key = None if keypack_data is not None: try: private_key = keypack_data['repo']['app_private'] except KeyError: # We will exit in _add_sig if private_key is None pass return private_key def _add_sig(self): # Adding new signature to version file # Raw private key will need to be converted into # a signing key object private_key_raw = self._load_private_keys() if private_key_raw is None: log.error('Private Key not found. Please ' 'import a keypack & try again') return update_data = self._load_update_data() if 'signature' in update_data: log.debug('Removing signatures from version file') del update_data['signature'] update_data_str = json.dumps(update_data, sort_keys=True) log.debug('Key type before: %s', type(private_key_raw)) private_key_raw = private_key_raw.encode('utf-8') log.debug('Key type after: %s', type(private_key_raw)) # Creating signing key object private_key = ed25519.SigningKey(private_key_raw, encoding=self.key_encoding) # Signs update data with private key signature = private_key.sign(six.b(update_data_str), encoding=self.key_encoding).decode() log.debug('Sig: %s', signature) update_data = json.loads(update_data_str) # Add signatures to update data update_data['signature'] = signature log.info('Adding sig to update data') # Write updated version file to filesystem self._write_update_data(update_data) self._write_key_file() def _write_update_data(self, data): # Save update data to repo database self.db.save(settings.CONFIG_DB_KEY_VERSION_META, data) log.debug('Saved version meta data') log.debug('Upload manifest: \n%s', data) # Gzip update date with gzip.open(self.version_file, 'wb') as f: new_data = json.dumps(data) if six.PY2: f.write(new_data) else: f.write(bytes(new_data, 'utf-8')) log.info('Created gzipped version manifest in deploy dir') def _write_key_file(self): keypack_data = self.db.load(settings.CONFIG_DB_KEY_KEYPACK) if keypack_data is None: log.error('Private Key not found. Please ' 'import a keypack & try again') return upload_data = keypack_data['upload'] with gzip.open(self.key_file, 'wb') as f: new_data = json.dumps(upload_data) if six.PY2: f.write(new_data) else: f.write(bytes(new_data, 'utf-8')) log.info('Created gzipped key file in deploy dir') def _load_update_data(self): log.debug("Loading version data") update_data = self.db.load(settings.CONFIG_DB_KEY_VERSION_META) # If update_data is None, create a new one if update_data is None: update_data = {} log.error('Version meta data not found') self.db.save(settings.CONFIG_DB_KEY_VERSION_META, update_data) log.info('Created new version meta data') log.debug('Version file loaded') return update_data
class PackageHandler(object): """Handles finding, sorting, getting meta-data, moving packages. Kwargs: app (instance): Config object """ def __init__(self, config=None): # References the pyu-data folder in the root of repo self.data_dir = None # Specifies if the config file needs to be loaded self.config_loaded = False # Used to store config information self.db = Storage() # Initialize app if config is available if config is not None: self.init_app(config) def init_app(self, obj): """Sets up client with config values from obj Args: obj (instance): config object """ self.patches = obj.get('UPDATE_PATCHES', True) if self.patches: log.debug('Patch support enabled') self.patch_support = True else: log.info('Patch support disabled') self.patch_support = False self.data_dir = os.path.join(os.getcwd(), settings.USER_DATA_FOLDER) self.files_dir = os.path.join(self.data_dir, 'files') self.deploy_dir = os.path.join(self.data_dir, 'deploy') self.new_dir = os.path.join(self.data_dir, 'new') self.config_dir = os.path.join(os.getcwd(), settings.CONFIG_DATA_FOLDER) self.config = None self.json_data = None self.setup() def setup(self): "Creates working directories & loads json files." if self.data_dir is not None: self._setup() def _setup(self): self._setup_work_dirs() if self.config_loaded is False: self.json_data = self._load_version_file() self.config = self._load_config() self.config_loaded = True def process_packages(self, report_errors=False): """Gets a list of updates to process. Adds the name of an update to the version file if not already present. Processes all packages. Updates the version file meta-data. Then writes version file back to disk. """ if self.data_dir is None: raise PackageHandlerError('Must init first.', expected=True) # Getting a list of meta data from all packages in the # pyu-data/new directory. Also create a patch manifest # to create patches. pkg_manifest, patch_manifest = self._get_package_list(report_errors) patches = self._make_patches(patch_manifest) self._cleanup(patch_manifest) pkg_manifest = self._add_patches_to_packages(pkg_manifest, patches) self.json_data = self._update_version_file(self.json_data, pkg_manifest) self._write_json_to_file(self.json_data) self._write_config_to_file(self.config) self._move_packages(pkg_manifest) def _setup_work_dirs(self): # Sets up work dirs on dev machine. Creates the following folder # - Data dir # Then inside the data folder it creates 3 more folders # - New - for new updates that need to be signed # - Deploy - All files ready to upload are placed here. # - Files - All updates are placed here for future reference # # This is non destructive dirs = [ self.data_dir, self.new_dir, self.deploy_dir, self.files_dir, self.config_dir ] for d in dirs: if not os.path.exists(d): log.info('Creating dir: %s', d) os.mkdir(d) def _load_version_file(self): # If version file is found its loaded to memory # If no version file is found then one is created. json_data = self.db.load(settings.CONFIG_DB_KEY_VERSION_META) if json_data is None: # pragma: no cover log.warning('Version file not found') json_data = {'updates': {}} log.info('Created new version file') return json_data def _load_config(self): # Loads config from db if exists. # If config doesn't exists create new one config = self.db.load(settings.CONFIG_DB_KEY_PY_REPO_CONFIG) if config is None: # pragma: no cover log.info('Creating new config file') config = {'patches': {}} return config def _get_package_list(self, report_errors): # Adds compatible packages to internal package manifest # for futher processing # Process all packages in new folder and gets # url, hash and some outer info. log.info('Getting package list') # Clears manifest if sign updates runs more the once without # app being restarted package_manifest = [] patch_manifest = [] bad_packages = [] with ChDir(self.new_dir): # Getting a list of all files in the new dir packages = os.listdir(os.getcwd()) for p in packages: # On package initialization we do the following # 1. Check for a supported archive # 2. get required info: version, platform, hash # If any check fails package.info['status'] will be False # You can query package.info['reason'] for the reason package = Package(p) if package.info['status'] is False: # Package failed at something # package.info['reason'] will tell why bad_packages.append(package) continue # Add package hash package.file_hash = gph(package.filename) package.file_size = in_bytes(package.filename) self.json_data = self._update_file_list( self.json_data, package) package_manifest.append(package) self.config = self._add_package_to_config(package, self.config) if self.patch_support: # If channel is not stable skip patch creation if package.channel != 'stable': log.debug( 'Package %s not on stable channel: ' 'Skipping', p) continue # Will check if source file for patch exists # if so will return the path and number of patch # to create. If missing source file None returned path = self._check_make_patch( self.json_data, package.name, package.platform, ) if path is not None: log.info('Found source file to create patch') patch_name = package.name + '-' + package.platform src_path = path[0] patch_number = path[1] patch_info = dict(src=src_path, dst=os.path.abspath(p), patch_name=os.path.join( self.new_dir, patch_name), patch_num=patch_number, package=package.filename) # ready for patching patch_manifest.append(patch_info) else: log.warning('No source file to patch from') # ToDo: Expose this & remove "pragma: no cover" once done if report_errors is True: # pragma: no cover log.warning('Bad package & reason for being naughty:') for b in bad_packages: log.warning(b.name, b.info['reason']) # End ToDo return package_manifest, patch_manifest def _add_package_to_config(self, p, data): if 'package' not in data.keys(): data['package'] = {} log.info('Initilizing config for packages') # First package with current name so add platform and version if p.name not in data['package'].keys(): data['package'][p.name] = {p.platform: p.version} log.info('Adding new package to config') else: # Adding platform and version if p.platform not in data['package'][p.name].keys(): data['package'][p.name][p.platform] = p.version log.info('Adding new arch to package-config: %s', p.platform) else: # Getting current version for platform value = data['package'][p.name][p.platform] # Updating version if applicable if p.version > value: log.info('Adding new version to package-config') data['package'][p.name][p.platform] = p.version return data def _cleanup(self, patch_manifest): # Remove old archives that were previously used to create patches if len(patch_manifest) < 1: return log.info('Cleaning up files directory') for p in patch_manifest: filename = os.path.basename(p['src']) directory = os.path.dirname(p['src']) remove_previous_versions(directory, filename) def _make_patches(self, patch_manifest): pool_output = [] if len(patch_manifest) < 1: return pool_output log.info('Starting patch creation') if sys.platform != 'win32': try: cpu_count = multiprocessing.cpu_count() * 2 except Exception as err: log.debug(err, exc_info=True) log.warning('Cannot get cpu count from os. Using default 2') cpu_count = 2 pool = multiprocessing.Pool(processes=cpu_count) pool_output = pool.map(_make_patch, patch_manifest) else: pool_output = [] for p in patch_manifest: pool_output.append(_make_patch(p)) return pool_output def _add_patches_to_packages(self, package_manifest, patches): if patches is not None and len(patches) >= 1: log.info('Adding patches to package list') for p in patches: # We'll skip if patch meta data is incomplete if hasattr(p, 'ready') is False: continue if hasattr(p, 'ready') and p.ready is False: continue for pm in package_manifest: # if p.dst_filename == pm.filename: pm.patch_info['patch_name'] = \ os.path.basename(p.patch_name) # Don't try to get hash on a ghost file if not os.path.exists(p.patch_name): p_name = '' p_size = 0 else: p_name = gph(p.patch_name) p_size = in_bytes(p.patch_name) pm.patch_info['patch_hash'] = p_name pm.patch_info['patch_size'] = p_size # No need to keep searching # We have the info we need for this patch break else: log.debug('No patch match found') else: if self.patch_support is True: log.warning('No patches found') return package_manifest def _update_file_list(self, json_data, package_info): files = json_data[settings.UPDATES_KEY] latest = json_data.get('latest') if latest is None: json_data['latest'] = {} filename = files.get(package_info.name) if filename is None: log.debug('Adding %s to file list', package_info.name) json_data[settings.UPDATES_KEY][package_info.name] = {} latest_package = json_data['latest'].get(package_info.name) if latest_package is None: json_data['latest'][package_info.name] = {} latest_package = json_data['latest'][package_info.name] latest_channel = latest_package.get(package_info.channel) if latest_channel is None: json_data['latest'][package_info.name][package_info.channel] = {} return json_data def _manifest_to_version_file_compat(self, package_info): # Checking for patch info. Patch info maybe be none patch_name = package_info.patch_info.get('patch_name') patch_hash = package_info.patch_info.get('patch_hash') patch_size = package_info.patch_info.get('patch_size') # Converting info to version file format info = { 'file_hash': package_info.file_hash, 'file_size': package_info.file_size, 'filename': package_info.filename } # Adding patch info if available if patch_name and patch_hash: info['patch_name'] = patch_name info['patch_hash'] = patch_hash info['patch_size'] = patch_size return info def _update_version_file(self, json_data, package_manifest): # Adding version metadata from scanned packages to our # version manifest log.info('Adding package meta-data to version manifest') easy_dict = EasyAccessDict(json_data) for p in package_manifest: info = self._manifest_to_version_file_compat(p) version_key = '{}*{}*{}'.format(settings.UPDATES_KEY, p.name, p.version) version = easy_dict.get(version_key) log.debug('Package Info: %s', version) # If we cannot get a version number this must be the first version # of its kind. if version is None: log.debug('Adding new version to file') # First version with this package name json_data[settings.UPDATES_KEY][p.name][p.version] = {} platform_key = '{}*{}*{}*{}'.format(settings.UPDATES_KEY, p.name, p.version, 'platform') platform = easy_dict.get(platform_key) if platform is None: _name = json_data[settings.UPDATES_KEY][p.name] _name[p.version][p.platform] = info else: # package already present, adding another version to it log.debug('Appending info data to version file') _updates = json_data[settings.UPDATES_KEY] _updates[p.name][p.version][p.platform] = info # Add each package to latests section separated by release channel json_data['latest'][p.name][p.channel][p.platform] = p.version return json_data def _write_json_to_file(self, json_data): # Writes json data to disk log.debug('Saving version meta-data') self.db.save(settings.CONFIG_DB_KEY_VERSION_META, json_data) def _write_config_to_file(self, json_data): log.debug('Saving config data') self.db.save(settings.CONFIG_DB_KEY_PY_REPO_CONFIG, json_data) def _move_packages(self, package_manifest): if len(package_manifest) < 1: return log.info('Moving packages to deploy folder') for p in package_manifest: patch = p.patch_info.get('patch_name') with ChDir(self.new_dir): if patch: if os.path.exists(os.path.join(self.deploy_dir, patch)): os.remove(os.path.join(self.deploy_dir, patch)) log.debug('Moving %s to %s', patch, self.deploy_dir) if os.path.exists(patch): shutil.move(patch, self.deploy_dir) shutil.copy(p.filename, self.deploy_dir) log.debug('Copying %s to %s', p.filename, self.deploy_dir) if os.path.exists(os.path.join(self.files_dir, p.filename)): os.remove(os.path.join(self.files_dir, p.filename)) shutil.move(p.filename, self.files_dir) log.debug('Moving %s to %s', p.filename, self.files_dir) def _check_make_patch(self, json_data, name, platform): # Check to see if previous version is available to # make patch updates. Also calculates patch number log.debug(json.dumps(json_data['latest'], indent=2)) log.info('Checking if patch creation is possible') if bsdiff4 is None: log.warning('Bsdiff is missing. Cannot create patches') return None src_file_path = None if os.path.exists(self.files_dir): with ChDir(self.files_dir): files = os.listdir(os.getcwd()) log.debug('Found %s files in files dir', len(files)) files = remove_dot_files(files) # No src files to patch from. Exit quickly if len(files) == 0: log.debug('No src file to patch from') return None # If latest not available in version file. Exit try: log.debug('Looking for %s on %s', name, platform) latest = json_data['latest'][name]['stable'][platform] log.debug('Found latest version for patches') except KeyError: log.debug('Cannot find latest version in version meta') return None try: latest_platform = json_data[settings.UPDATES_KEY][name][latest] log.debug('Found latest platform for patches') try: filename = latest_platform[platform]['filename'] log.debug('Found filename for patches') except KeyError: log.error('Found old version file. Please read ' 'the upgrade section in the docs.') log.debug('Found old verison file') return None except Exception as err: log.debug(err, exc_info=True) return None log.debug('Generating src file path') src_file_path = os.path.join(self.files_dir, filename) try: patch_num = self.config['patches'][name] log.debug('Found patch number') self.config['patches'][name] += 1 except KeyError: log.debug('Cannot find patch number') # If no patch number we will start at 1 patch_num = 1 if 'patches' not in self.config.keys(): log.debug('Adding patches to version meta') self.config['patches'] = {} if name not in self.config['patches'].keys(): log.debug('Adding %s to patches version meta', name) self.config['patches'][name] = patch_num + 1 num = patch_num + 1 log.debug('Patch Number: %s', num) return src_file_path, num return None
class KeyHandler(object): """KeyHandler object is used to manage keys used for signing updates Kwargs: app (obj): Config object to get config values from """ def __init__(self): self.db = Storage() self.key_encoding = "base64" data_dir = os.getcwd() self.data_dir = os.path.join(data_dir, settings.USER_DATA_FOLDER) self.deploy_dir = os.path.join(self.data_dir, "deploy") # Name of the keypack to import. It should be placed # in the root of the repo self.keypack_filename = os.path.join( data_dir, settings.CONFIG_DATA_FOLDER, settings.KEYPACK_FILENAME ) # The name of the gzipped version file in # the pyu-data/deploy directory self.version_file = os.path.join( self.deploy_dir, settings.VERSION_FILE_FILENAME ) self.version_file_compat = os.path.join( self.deploy_dir, settings.VERSION_FILE_FILENAME_COMPAT ) # The name of the gzipped key file in # the pyu-data/deploy directory self.key_file = os.path.join(self.deploy_dir, settings.KEY_FILE_FILENAME) def sign_update(self, split_version): """Signs version file with private key Proxy method for :meth:`_add_sig` """ # Loads private key # Loads version file to memory # Signs Version file # Writes version file back to disk self._add_sig(split_version) def _load_private_keys(self): # Loads private key log.debug("Loading private key") # Loading keypack data from .pyupdater/config.pyu keypack_data = self.db.load(settings.CONFIG_DB_KEY_KEYPACK) private_key = None if keypack_data is not None: try: private_key = keypack_data["repo"]["app_private"] except KeyError: # We will exit in _add_sig if private_key is None pass return private_key def _add_sig(self, split_version): # Adding new signature to version file # Raw private key will need to be converted into # a signing key object private_key_raw = self._load_private_keys() if private_key_raw is None: log.error("Private Key not found. Please " "import a keypack & try again") return # Load update manifest update_data = self._load_update_data() # We don't want to verify the signature if "signature" in update_data: log.debug("Removing signatures from version file") del update_data["signature"] # We create a signature from the string update_data_str = json.dumps(update_data, sort_keys=True) private_key_raw = private_key_raw.encode("utf-8") # Creating signing key object private_key = ed25519.SigningKey(private_key_raw, encoding=self.key_encoding) log.debug("Signing update data") # Signs update data with private key signature = private_key.sign( bytes(update_data_str, 'latin-1'), encoding=self.key_encoding ).decode() log.debug("Sig: %s", signature) # Create new dict from json string update_data = json.loads(update_data_str) # Add signatures to update data update_data["signature"] = signature log.debug("Adding signature to update data") # Write updated version file to .pyupdater/config.pyu self._write_update_data(update_data, split_version) # Write gzipped key file self._write_key_file() def _write_update_data(self, data, split_version): log.debug("Saved version meta data") if split_version: version_file = self.version_file else: version_file = self.version_file_compat # Gzip update date with gzip.open(version_file, "wb") as f: new_data = json.dumps(data) f.write(bytes(new_data, "utf-8")) log.debug("Created gzipped version manifest in deploy dir") def _write_key_file(self): keypack_data = self.db.load(settings.CONFIG_DB_KEY_KEYPACK) if keypack_data is None: log.error("Private Key not found. Please import a keypack & try again") return upload_data = keypack_data["upload"] with gzip.open(self.key_file, "wb") as f: new_data = json.dumps(upload_data) f.write(bytes(new_data, "utf-8")) log.debug("Created gzipped key file in deploy dir") def _load_update_data(self): log.debug("Loading version data") update_data = self.db.load(settings.CONFIG_DB_KEY_VERSION_META) # If update_data is None, create a new one if update_data is None: update_data = {} log.error("Version meta data not found") self.db.save(settings.CONFIG_DB_KEY_VERSION_META, update_data) log.debug("Created new version meta data") log.debug("Version file loaded") return copy.deepcopy(update_data)
class Loader(object): """Loads & saves config file """ def __init__(self): self.cwd = os.getcwd() self.db = Storage() self.password = os.environ.get(settings.USER_PASS_ENV) self.config_key = settings.CONFIG_DB_KEY_APP_CONFIG def load_config(self): """Loads config from database (json file) Returns (obj): Config object """ config_data = self.db.load(self.config_key) if config_data is None: config_data = {} config = Config() for k, v in config_data.items(): config[k] = v config.DATA_DIR = os.getcwd() return config def get_app_name(self): config = self.load_config() return config.APP_NAME def save_config(self, obj): """Saves config file to pyupdater database Args: obj (obj): config object """ log.info('Saving Config') self.db.save(self.config_key, obj) log.info('Config saved') self._write_config_py(obj) log.info('Wrote client config') def _write_config_py(self, obj): """Writes client config to client_config.py Args: obj (obj): config object """ keypack_data = self.db.load(settings.CONFIG_DB_KEY_KEYPACK) if keypack_data is None: public_key = None else: public_key = keypack_data['client']['offline_public'] filename = os.path.join(self.cwd, *obj.CLIENT_CONFIG_PATH) attr_str_format = " {} = '{}'\n" attr_format = " {} = {}\n" with open(filename, 'w') as f: f.write('class ClientConfig(object):\n') if hasattr(obj, 'APP_NAME') and obj.APP_NAME is not None: f.write(attr_str_format.format('APP_NAME', obj.APP_NAME)) log.debug('Wrote APP_NAME to client config') if hasattr(obj, 'COMPANY_NAME') and obj.COMPANY_NAME is not None: f.write(attr_str_format.format('COMPANY_NAME', obj.COMPANY_NAME)) log.debug('Wrote COMPANY_NAME to client config') if hasattr(obj, 'UPDATE_URLS') and obj.UPDATE_URLS is not None: f.write(attr_format.format('UPDATE_URLS', obj.UPDATE_URLS)) log.debug('Wrote UPDATE_URLS to client config') f.write(attr_str_format.format('PUBLIC_KEY', public_key)) log.debug('Wrote PUBLIC_KEY to client config')
class ConfigManager(object): def __init__(self): self.cwd = os.getcwd() self.db = Storage() self.config_key = settings.CONFIG_DB_KEY_APP_CONFIG # Loads config from database (json file) def load_config(self): config_data = self.db.load(self.config_key) if config_data is None: config_data = {} config = Config() for k, v in config_data.items(): config[k] = v config.DATA_DIR = os.getcwd() return config def get_app_name(self): config = self.load_config() return config.APP_NAME # Saves config to database (json file) def save_config(self, obj): log.debug("Saving Config") self.db.save(self.config_key, obj) log.debug("Config saved") self.write_config_py(obj) log.debug("Wrote client config") # Writes client config to client_config.py def write_config_py(self, obj): keypack_data = self.db.load(settings.CONFIG_DB_KEY_KEYPACK) if keypack_data is None: log.debug("*** Keypack data is None ***") public_key = None else: public_key = keypack_data["client"]["offline_public"] filename = os.path.join(self.cwd, *obj.CLIENT_CONFIG_PATH) attr_str_format = " {} = '{}'\n" attr_format = " {} = {}\n" log.debug("Writing client_config.py") with open(filename, "w") as f: f.write("class ClientConfig(object):\n") log.debug("Adding PUBLIC_KEY to client_config.py") f.write(attr_str_format.format("PUBLIC_KEY", public_key)) if hasattr(obj, "APP_NAME"): log.debug("Adding APP_NAME to client_config.py") f.write(attr_str_format.format("APP_NAME", obj.APP_NAME)) if hasattr(obj, "COMPANY_NAME"): log.debug("Adding COMPANY_NAME to client_config.py") f.write( attr_str_format.format("COMPANY_NAME", obj.COMPANY_NAME)) if hasattr(obj, "HTTP_TIMEOUT"): log.debug("Adding HTTP_TIMEOUT to cilent_config.py") f.write(attr_format.format("HTTP_TIMEOUT", obj.HTTP_TIMEOUT)) if hasattr(obj, "MAX_DOWNLOAD_RETRIES"): log.debug("Adding MAX_DOWNLOAD_RETRIES to client_config.py") f.write( attr_format.format("MAX_DOWNLOAD_RETRIES", obj.MAX_DOWNLOAD_RETRIES)) if hasattr(obj, "UPDATE_URLS"): log.debug("Adding UPDATE_URLS to client_config.py") f.write(attr_format.format("UPDATE_URLS", obj.UPDATE_URLS))
class PackageHandler(object): """Handles finding, sorting, getting meta-data, moving packages. Kwargs: app (instance): Config object """ def __init__(self, config=None): # Configuration data self.config = None # Version manifest file self.version_data = None # Specifies if the config file needs to be loaded self.config_loaded = False # Used to store config information self.db = Storage() if config: # Support for creating patches self.patch_support = config.get("UPDATE_PATCHES", True) is True else: self.patch_support = False # References the pyu-data folder in the root of repo self.data_dir = os.path.join(os.getcwd(), settings.USER_DATA_FOLDER) self.files_dir = os.path.join(self.data_dir, "files") self.deploy_dir = os.path.join(self.data_dir, "deploy") self.new_dir = os.path.join(self.data_dir, "new") self.config_dir = os.path.join(os.getcwd(), settings.CONFIG_DATA_FOLDER) self.setup() def setup(self): """Creates working directories & loads json files.""" if self.data_dir is not None: self._setup() def _setup(self): self._setup_work_dirs() if self.config_loaded is False: self.version_data = self._load_version_file() self.config = self._load_config() self.config_loaded = True def process_packages(self, report_errors=False): """Gets a list of updates to process. Adds the name of an update to the version file if not already present. Processes all packages. Updates the version file meta-data. Then writes version file back to disk. """ if self.patch_support: log.info("Patch support enabled") else: log.info("Patch support disabled") # Getting a list of meta data from all packages in the # pyu-data/new directory. Also create a patch manifest # to create patches. pkg_manifest, patch_manifest = self._get_package_list(report_errors) patches = PackageHandler._make_patches(patch_manifest) PackageHandler._cleanup(patch_manifest) PackageHandler._add_patches_to_packages(pkg_manifest, patches, self.patch_support) PackageHandler._update_version_file(self.version_data, pkg_manifest) self._write_json_to_file(self.version_data) self._write_config_to_file(self.config) self._move_packages(pkg_manifest) def _setup_work_dirs(self): # Sets up work dirs on dev machine. Creates the following folder # - Data dir # Then inside the data folder it creates 3 more folders # - New - for new updates that need to be signed # - Deploy - All files ready to upload are placed here. # - Files - All updates are placed here for future reference # # This is non destructive dirs = [ self.data_dir, self.new_dir, self.deploy_dir, self.files_dir, self.config_dir, ] for d in dirs: if not os.path.exists(d): log.info("Creating dir: %s", d) os.mkdir(d) def _load_version_file(self): # If version file is found its loaded to memory # If no version file is found then one is created. json_data = self.db.load(settings.CONFIG_DB_KEY_VERSION_META) if json_data is None: # pragma: no cover log.warning("Version file not found") json_data = {"updates": {}} log.debug("Created new version file") return json_data def _load_config(self): # Loads config from db if exists. # If config doesn't exists create new one config = self.db.load(settings.CONFIG_DB_KEY_PY_REPO_CONFIG) if config is None: # pragma: no cover log.debug("Creating new config file") config = {"patches": {}} return config def _get_package_list(self, report_errors): # Adds compatible packages to internal package manifest # for further processing # Process all packages in new folder and gets # url, hash and some outer info. log.info("Generating package list") # Clears manifest if sign updates runs more the once without # app being restarted package_manifest = [] patch_manifest = [] bad_packages = [] with ChDir(self.new_dir): # Getting a list of all files in the new dir packages = os.listdir(os.getcwd()) for p in packages: # On package initialization we do the following # 1. Check for a supported archive # 2. get required info: version, platform, hash # If any check fails new_pkg.info['status'] will be False # You can query new_pkg.info['reason'] for the reason new_pkg = Package(p) if new_pkg.info["status"] is False: # Package failed at something # new_pkg.info['reason'] will tell why bad_packages.append(new_pkg) continue # Add package hash new_pkg.file_hash = gph(new_pkg.filename) new_pkg.file_size = in_bytes(new_pkg.filename) PackageHandler._update_file_list(self.version_data, new_pkg) package_manifest.append(new_pkg) PackageHandler._add_package_to_config(new_pkg, self.config) if self.patch_support: data = { "filename": p, "files_dir": self.files_dir, "new_dir": self.new_dir, "json_data": self.version_data, "pkg_info": new_pkg, "config": self.config, } _patch = Patch(**data) if _patch.ok: patch_manifest.append(_patch) if report_errors is True: # pragma: no cover log.warning("Bad package & reason for being naughty:") for b in bad_packages: log.warning(b.name, b.info["reason"]) return package_manifest, patch_manifest @staticmethod def _add_package_to_config(p, data): if "package" not in data.keys(): data["package"] = {} log.debug("Initializing config for packages") # First package with current name so add platform and version if p.name not in data["package"].keys(): data["package"][p.name] = {p.platform: p.version} log.debug("Adding new package to config") else: # Adding platform and version if p.platform not in data["package"][p.name].keys(): data["package"][p.name][p.platform] = p.version log.debug("Adding new arch to package-config: %s", p.platform) else: # Getting current version for platform value = data["package"][p.name][p.platform] # Updating version if applicable if p.version > value: log.debug("Adding new version to package-config") data["package"][p.name][p.platform] = p.version @staticmethod def _cleanup(patch_manifest): # Remove old archives that were previously used to create patches if len(patch_manifest) < 1: return log.info("Cleaning up stale files") for p in patch_manifest: remove_previous_versions(os.path.dirname(p.src), p.dst) @staticmethod def _make_patches(patch_manifest): pool_output = [] if len(patch_manifest) < 1: return pool_output log.info("Starting patch creation") if sys.platform != "win32": try: cpu_count = multiprocessing.cpu_count() * 2 except Exception as err: log.debug(err, exc_info=True) log.warning("Cannot get cpu count from os. Using default 2") cpu_count = 2 pool = multiprocessing.Pool(processes=cpu_count) pool_output = pool.map(make_patch, patch_manifest) else: pool_output = [] for p in patch_manifest: pool_output.append(make_patch(p)) return pool_output @staticmethod def _add_patches_to_packages(package_manifest, patches, patch_support): if patches is not None and len(patches) >= 1: log.debug("Adding patches to package list") for p in patches: if not p.ok or not os.path.exists(p.patch_name): continue log.debug("We have a good patch: %s", p) for pm in package_manifest: if p.dst_filename == pm.filename: pm.patch = p pm.patch.hash = gph(pm.patch.patch_name) pm.patch.size = in_bytes(pm.patch.patch_name) break else: log.debug("No patch match found") else: if patch_support is True: log.debug("No patches found: %s", patches) @staticmethod def _update_file_list(json_data, package_info): files = json_data[settings.UPDATES_KEY] latest = json_data.get("latest") if latest is None: json_data["latest"] = {} filename = files.get(package_info.name) if filename is None: log.debug("Adding %s to file list", package_info.name) json_data[settings.UPDATES_KEY][package_info.name] = {} latest_package = json_data["latest"].get(package_info.name) if latest_package is None: json_data["latest"][package_info.name] = {} latest_package = json_data["latest"][package_info.name] latest_channel = latest_package.get(package_info.channel) if latest_channel is None: json_data["latest"][package_info.name][package_info.channel] = {} return json_data @staticmethod def _manifest_to_version_file_compat(package_info): # Converting info to version file format info = { "file_hash": package_info.file_hash, "file_size": package_info.file_size, "filename": package_info.filename, } # Adding patch info if available if package_info.patch is not None: info["patch_name"] = package_info.patch.basename info["patch_hash"] = package_info.patch.hash info["patch_size"] = package_info.patch.size return info @staticmethod def _update_version_file(json_data, package_manifest): # Adding version metadata from scanned packages to our # version manifest log.info("Adding package meta-data to version manifest") easy_dict = EasyAccessDict(json_data) for p in package_manifest: info = PackageHandler._manifest_to_version_file_compat(p) version_key = "{}*{}*{}".format(settings.UPDATES_KEY, p.name, p.version) version = easy_dict.get(version_key) log.debug("Package Info: %s", version) # If we cannot get a version number this must be the first version # of its kind. if version is None: log.debug("Adding new version to file") # First version with this package name json_data[settings.UPDATES_KEY][p.name][p.version] = {} platform_key = "{}*{}*{}*{}".format(settings.UPDATES_KEY, p.name, p.version, "platform") platform = easy_dict.get(platform_key) if platform is None: _name = json_data[settings.UPDATES_KEY][p.name] _name[p.version][p.platform] = info else: # package already present, adding another version to it log.debug("Appending info data to version file") _updates = json_data[settings.UPDATES_KEY] _updates[p.name][p.version][p.platform] = info # Add each package to latest section separated by release channel json_data["latest"][p.name][p.channel][p.platform] = p.version return json_data def _write_json_to_file(self, json_data): # Writes json data to disk log.debug("Saving version meta-data") self.db.save(settings.CONFIG_DB_KEY_VERSION_META, json_data) def _write_config_to_file(self, json_data): log.debug("Saving config data") self.db.save(settings.CONFIG_DB_KEY_PY_REPO_CONFIG, json_data) def _move_packages(self, package_manifest): if len(package_manifest) < 1: return log.info("Moving packages to deploy folder") for p in package_manifest: with ChDir(self.new_dir): if p.patch is not None: if os.path.exists( os.path.join(self.deploy_dir, p.patch.basename)): os.remove( os.path.join(self.deploy_dir, p.patch.basename)) log.debug("Moving %s to %s", p.patch.basename, self.deploy_dir) if os.path.exists(p.patch.basename): shutil.move(p.patch.basename, self.deploy_dir) shutil.copy(p.filename, self.deploy_dir) log.debug("Copying %s to %s", p.filename, self.deploy_dir) if os.path.exists(os.path.join(self.files_dir, p.filename)): os.remove(os.path.join(self.files_dir, p.filename)) shutil.move(p.filename, self.files_dir) log.debug("Moving %s to %s", p.filename, self.files_dir)
class PackageHandler(object): """Handles finding, sorting, getting meta-data, moving packages. Kwargs: app (instance): Config object """ def __init__(self, config=None): # Configuration data self.config = None # Version manifest file self.version_data = None # Specifies if the config file needs to be loaded self.config_loaded = False # Used to store config information self.db = Storage() if config: # Support for creating patches self.patch_support = config.get('UPDATE_PATCHES', True) is True else: self.patch_support = False # References the pyu-data folder in the root of repo self.data_dir = os.path.join(os.getcwd(), settings.USER_DATA_FOLDER) self.files_dir = os.path.join(self.data_dir, 'files') self.deploy_dir = os.path.join(self.data_dir, 'deploy') self.new_dir = os.path.join(self.data_dir, 'new') self.config_dir = os.path.join(os.getcwd(), settings.CONFIG_DATA_FOLDER) self.setup() def setup(self): """Creates working directories & loads json files.""" if self.data_dir is not None: self._setup() def _setup(self): self._setup_work_dirs() if self.config_loaded is False: self.version_data = self._load_version_file() self.config = self._load_config() self.config_loaded = True def process_packages(self, report_errors=False): """Gets a list of updates to process. Adds the name of an update to the version file if not already present. Processes all packages. Updates the version file meta-data. Then writes version file back to disk. """ if self.patch_support: log.info('Patch support enabled') else: log.info('Patch support disabled') # Getting a list of meta data from all packages in the # pyu-data/new directory. Also create a patch manifest # to create patches. pkg_manifest, patch_manifest = self._get_package_list(report_errors) patches = PackageHandler._make_patches(patch_manifest) PackageHandler._cleanup(patch_manifest) pkg_manifest = self._add_patches_to_packages(pkg_manifest, patches) # PEP8 json_data = PackageHandler._update_version_file(self.version_data, pkg_manifest) self.version_data = json_data self._write_json_to_file(self.version_data) self._write_config_to_file(self.config) self._move_packages(pkg_manifest) def _setup_work_dirs(self): # Sets up work dirs on dev machine. Creates the following folder # - Data dir # Then inside the data folder it creates 3 more folders # - New - for new updates that need to be signed # - Deploy - All files ready to upload are placed here. # - Files - All updates are placed here for future reference # # This is non destructive dirs = [self.data_dir, self.new_dir, self.deploy_dir, self.files_dir, self.config_dir] for d in dirs: if not os.path.exists(d): log.info('Creating dir: %s', d) os.mkdir(d) def _load_version_file(self): # If version file is found its loaded to memory # If no version file is found then one is created. json_data = self.db.load(settings.CONFIG_DB_KEY_VERSION_META) if json_data is None: # pragma: no cover log.warning('Version file not found') json_data = {'updates': {}} log.debug('Created new version file') return json_data def _load_config(self): # Loads config from db if exists. # If config doesn't exists create new one config = self.db.load(settings.CONFIG_DB_KEY_PY_REPO_CONFIG) if config is None: # pragma: no cover log.debug('Creating new config file') config = { 'patches': {} } return config def _get_package_list(self, report_errors): # Adds compatible packages to internal package manifest # for futher processing # Process all packages in new folder and gets # url, hash and some outer info. log.info('Generating package list') # Clears manifest if sign updates runs more the once without # app being restarted package_manifest = [] patch_manifest = [] bad_packages = [] with ChDir(self.new_dir): # Getting a list of all files in the new dir packages = os.listdir(os.getcwd()) for p in packages: # On package initialization we do the following # 1. Check for a supported archive # 2. get required info: version, platform, hash # If any check fails package.info['status'] will be False # You can query package.info['reason'] for the reason package = Package(p) if package.info['status'] is False: # Package failed at something # package.info['reason'] will tell why bad_packages.append(package) continue # Add package hash package.file_hash = gph(package.filename) package.file_size = in_bytes(package.filename) self.version_data = PackageHandler._update_file_list(self.version_data, package) package_manifest.append(package) self.config = PackageHandler._add_package_to_config(package, self.config) if self.patch_support: # If channel is not stable skip patch creation if package.channel != 'stable': log.debug('Package %s not on stable channel: ' 'Skipping', p) continue # Will check if source file for patch exists # if so will return the path and number of patch # to create. If missing source file None returned path = self._check_make_patch(self.version_data, package.name, package.platform, ) if path is not None: log.debug('Found source file to create patch') patch_name = package.name + '-' + package.platform src_path = path[0] patch_number = path[1] patch_info = dict(src=src_path, dst=os.path.abspath(p), patch_name=os.path.join(self.new_dir, patch_name), patch_num=patch_number, package=package.filename) # ready for patching patch_manifest.append(patch_info) else: log.warning('No source file to patch from') if report_errors is True: # pragma: no cover log.warning('Bad package & reason for being naughty:') for b in bad_packages: log.warning(b.name, b.info['reason']) return package_manifest, patch_manifest @staticmethod def _add_package_to_config(p, data): if 'package' not in data.keys(): data['package'] = {} log.debug('Initilizing config for packages') # First package with current name so add platform and version if p.name not in data['package'].keys(): data['package'][p.name] = {p.platform: p.version} log.debug('Adding new package to config') else: # Adding platform and version if p.platform not in data['package'][p.name].keys(): data['package'][p.name][p.platform] = p.version log.debug('Adding new arch to package-config: %s', p.platform) else: # Getting current version for platform value = data['package'][p.name][p.platform] # Updating version if applicable if p.version > value: log.debug('Adding new version to package-config') data['package'][p.name][p.platform] = p.version return data @staticmethod def _cleanup(patch_manifest): # Remove old archives that were previously used to create patches if len(patch_manifest) < 1: return log.info('Cleaning up stale files') for p in patch_manifest: filename = os.path.basename(p['src']) directory = os.path.dirname(p['src']) remove_previous_versions(directory, filename) @staticmethod def _make_patches(patch_manifest): pool_output = [] if len(patch_manifest) < 1: return pool_output log.info('Starting patch creation') if sys.platform != 'win32': try: cpu_count = multiprocessing.cpu_count() * 2 except Exception as err: log.debug(err, exc_info=True) log.warning('Cannot get cpu count from os. Using default 2') cpu_count = 2 pool = multiprocessing.Pool(processes=cpu_count) pool_output = pool.map(_make_patch, patch_manifest) else: pool_output = [] for p in patch_manifest: pool_output.append(_make_patch(p)) return pool_output def _add_patches_to_packages(self, package_manifest, patches): if patches is not None and len(patches) >= 1: log.debug('Adding patches to package list') for p in patches: # We'll skip if patch meta data is incomplete if hasattr(p, 'ready') is False: continue if hasattr(p, 'ready') and p.ready is False: continue for pm in package_manifest: # if p.dst_filename == pm.filename: pm.patch_info['patch_name'] = \ os.path.basename(p.patch_name) # Don't try to get hash on a ghost file if not os.path.exists(p.patch_name): p_name = '' p_size = 0 else: p_name = gph(p.patch_name) p_size = in_bytes(p.patch_name) pm.patch_info['patch_hash'] = p_name pm.patch_info['patch_size'] = p_size # No need to keep searching # We have the info we need for this patch break else: log.debug('No patch match found') else: if self.patch_support is True: log.debug('No patches found') return package_manifest @staticmethod def _update_file_list(json_data, package_info): files = json_data[settings.UPDATES_KEY] latest = json_data.get('latest') if latest is None: json_data['latest'] = {} filename = files.get(package_info.name) if filename is None: log.debug('Adding %s to file list', package_info.name) json_data[settings.UPDATES_KEY][package_info.name] = {} latest_package = json_data['latest'].get(package_info.name) if latest_package is None: json_data['latest'][package_info.name] = {} latest_package = json_data['latest'][package_info.name] latest_channel = latest_package.get(package_info.channel) if latest_channel is None: json_data['latest'][package_info.name][package_info.channel] = {} return json_data @staticmethod def _manifest_to_version_file_compat(package_info): # Checking for patch info. Patch info maybe be none patch_name = package_info.patch_info.get('patch_name') patch_hash = package_info.patch_info.get('patch_hash') patch_size = package_info.patch_info.get('patch_size') # Converting info to version file format info = { 'file_hash': package_info.file_hash, 'file_size': package_info.file_size, 'filename': package_info.filename } # Adding patch info if available if patch_name and patch_hash: info['patch_name'] = patch_name info['patch_hash'] = patch_hash info['patch_size'] = patch_size return info @staticmethod def _update_version_file(json_data, package_manifest): # Adding version metadata from scanned packages to our # version manifest log.info('Adding package meta-data to version manifest') easy_dict = EasyAccessDict(json_data) for p in package_manifest: info = PackageHandler._manifest_to_version_file_compat(p) version_key = '{}*{}*{}'.format(settings.UPDATES_KEY, p.name, p.version) version = easy_dict.get(version_key) log.debug('Package Info: %s', version) # If we cannot get a version number this must be the first version # of its kind. if version is None: log.debug('Adding new version to file') # First version with this package name json_data[settings.UPDATES_KEY][p.name][p.version] = {} platform_key = '{}*{}*{}*{}'.format(settings.UPDATES_KEY, p.name, p.version, 'platform') platform = easy_dict.get(platform_key) if platform is None: _name = json_data[settings.UPDATES_KEY][p.name] _name[p.version][p.platform] = info else: # package already present, adding another version to it log.debug('Appending info data to version file') _updates = json_data[settings.UPDATES_KEY] _updates[p.name][p.version][p.platform] = info # Add each package to latest section separated by release channel json_data['latest'][p.name][p.channel][p.platform] = p.version return json_data def _write_json_to_file(self, json_data): # Writes json data to disk log.debug('Saving version meta-data') self.db.save(settings.CONFIG_DB_KEY_VERSION_META, json_data) def _write_config_to_file(self, json_data): log.debug('Saving config data') self.db.save(settings.CONFIG_DB_KEY_PY_REPO_CONFIG, json_data) def _move_packages(self, package_manifest): if len(package_manifest) < 1: return log.info('Moving packages to deploy folder') for p in package_manifest: patch = p.patch_info.get('patch_name') with ChDir(self.new_dir): if patch: if os.path.exists(os.path.join(self.deploy_dir, patch)): os.remove(os.path.join(self.deploy_dir, patch)) log.debug('Moving %s to %s', patch, self.deploy_dir) if os.path.exists(patch): shutil.move(patch, self.deploy_dir) shutil.copy(p.filename, self.deploy_dir) log.debug('Copying %s to %s', p.filename, self.deploy_dir) if os.path.exists(os.path.join(self.files_dir, p.filename)): os.remove(os.path.join(self.files_dir, p.filename)) shutil.move(p.filename, self.files_dir) log.debug('Moving %s to %s', p.filename, self.files_dir) def _check_make_patch(self, json_data, name, platform): # Check to see if previous version is available to # make patch updates. Also calculates patch number log.debug(json.dumps(json_data['latest'], indent=2)) log.debug('Checking if patch creation is possible') if bsdiff4 is None: log.warning('Bsdiff is missing. Cannot create patches') return None if os.path.exists(self.files_dir): with ChDir(self.files_dir): files = os.listdir(os.getcwd()) log.debug('Found %s files in files dir', len(files)) files = remove_dot_files(files) # No src files to patch from. Exit quickly if len(files) == 0: log.debug('No src file to patch from') return None # If latest not available in version file. Exit try: log.debug('Looking for %s on %s', name, platform) latest = json_data['latest'][name]['stable'][platform] log.debug('Found latest version for patches') except KeyError: log.debug('Cannot find latest version in version meta') return None try: latest_platform = json_data[settings.UPDATES_KEY][name][latest] log.debug('Found latest platform for patches') try: filename = latest_platform[platform]['filename'] log.debug('Found filename for patches') except KeyError: log.error('Found old version file. Please read ' 'the upgrade section in the docs.') log.debug('Found old verison file') return None except Exception as err: log.debug(err, exc_info=True) return None log.debug('Generating src file path') src_file_path = os.path.join(self.files_dir, filename) try: patch_num = self.config['patches'][name] log.debug('Found patch number') self.config['patches'][name] += 1 except KeyError: log.debug('Cannot find patch number') # If no patch number we will start at 1 patch_num = 1 if 'patches' not in self.config.keys(): log.debug('Adding patches to version meta') self.config['patches'] = {} if name not in self.config['patches'].keys(): log.debug('Adding %s to patches version meta', name) self.config['patches'][name] = patch_num + 1 num = patch_num + 1 log.debug('Patch Number: %s', num) return src_file_path, num return None
class PackageHandler(object): """Handles finding, sorting, getting meta-data, moving packages. Kwargs: app (instance): Config object """ data_dir = None def __init__(self, app=None): self.config_loaded = False self.db = Storage() if app is not None: self.init_app(app) def init_app(self, obj): """Sets up client with config values from obj Args: obj (instance): config object """ self.patches = obj.get('UPDATE_PATCHES', True) if self.patches: log.debug('Patch support enabled') self.patch_support = True else: log.info('Patch support disabled') self.patch_support = False data_dir = os.getcwd() self.data_dir = os.path.join(data_dir, settings.USER_DATA_FOLDER) self.files_dir = os.path.join(self.data_dir, 'files') self.deploy_dir = os.path.join(self.data_dir, 'deploy') self.new_dir = os.path.join(self.data_dir, 'new') self.config_dir = os.path.join(os.path.dirname(self.data_dir), settings.CONFIG_DATA_FOLDER) self.config = None self.json_data = None self.setup() def setup(self): "Creates working directories & loads json files." if self.data_dir is not None: self._setup() def _setup(self): self._setup_work_dirs() if self.config_loaded is False: self.json_data = self._load_version_file() self.config = self._load_config() self.config_loaded = True def process_packages(self): """Gets a list of updates to process. Adds the name of an update to the version file if not already present. Processes all packages. Updates the version file meta-data. Then writes version file back to disk. """ if self.data_dir is None: raise PackageHandlerError('Must init first.', expected=True) package_manifest, patch_manifest = self._get_package_list() patches = self._make_patches(patch_manifest) self._cleanup(patch_manifest) package_manifest = self._add_patches_to_packages(package_manifest, patches) self.json_data = self._update_version_file(self.json_data, package_manifest) self._write_json_to_file(self.json_data) self._write_config_to_file(self.config) self._move_packages(package_manifest) def _setup_work_dirs(self): # Sets up work dirs on dev machine. Creates the following folder # - Data dir # Then inside the data folder it creates 3 more folders # - New - for new updates that need to be signed # - Deploy - All files ready to upload are placed here. # - Files - All updates are placed here for future reference # # This is non destructive dirs = [self.data_dir, self.new_dir, self.deploy_dir, self.files_dir, self.config_dir] for d in dirs: if not os.path.exists(d): log.info('Creating dir: {}'.format(d)) os.mkdir(d) def _load_version_file(self): # If version file is found its loaded to memory # If no version file is found then one is created. json_data = self.db.load(settings.CONFIG_DB_KEY_VERSION_META) if json_data is None: # pragma: no cover log.warning('Version file not found') json_data = {'updates': {}} log.info('Created new version file') return json_data def _load_config(self): # Loads config from db if exists. # If config doesn't exists create new one config = self.db.load(settings.CONFIG_DB_KEY_PY_REPO_CONFIG) if config is None: # pragma: no cover log.info('Creating new config file') config = { 'patches': {} } return config def _get_package_list(self, ignore_errors=True): # Adds compatible packages to internal package manifest # for futher processing # Process all packages in new folder and gets # url, hash and some outer info. log.info('Getting package list') # Clears manifest if sign updates runs more the once without # app being restarted package_manifest = list() patch_manifest = list() bad_packages = list() with jms_utils.paths.ChDir(self.new_dir): # Getting a list of all files in the new dir packages = os.listdir(os.getcwd()) for p in packages: # On package initialization we do the following # 1. Check for a supported archive # 2. get required info: version, platform, hash # If any check fails package.info['status'] will be False # You can query package.info['reason'] for the reason package = Package(p) if package.info['status'] is False: # Package failed at something # package.info['reason'] will tell why bad_packages.append(package) continue # Add package hash package.file_hash = gph(package.filename) package.file_size = in_bytes(package.filename) self.json_data = self._update_file_list(self.json_data, package) package_manifest.append(package) self.config = self._add_package_to_config(package, self.config) if self.patch_support: # If channel is not stable skip patch creation if package.channel != 'stable': log.debug('Package {} not on stable channel: ' 'Skipping'.format(p)) continue # Will check if source file for patch exists # if so will return the path and number of patch # to create. If missing source file None returned path = self._check_make_patch(self.json_data, package.name, package.platform, ) if path is not None: log.info('Found source file to create patch') patch_name = package.name + '-' + package.platform src_path = path[0] patch_number = path[1] patch_info = dict(src=src_path, dst=os.path.abspath(p), patch_name=os.path.join(self.new_dir, patch_name), patch_num=patch_number, package=package.filename) # ready for patching patch_manifest.append(patch_info) else: log.warning('No source file to patch from') # ToDo: Expose this & remove "pragma: no cover" once done if ignore_errors is False: # pragma: no cover log.warning('Bad package & reason for being naughty:') for b in bad_packages: log.warning(b.name, b.info['reason']) # End ToDo return package_manifest, patch_manifest def _add_package_to_config(self, p, data): if 'package' not in data.keys(): data['package'] = {} log.info('Initilizing config for packages') # First package with current name so add platform and version if p.name not in data['package'].keys(): data['package'][p.name] = {p.platform: p.version} log.info('Adding new package to config') else: # Adding platform and version if p.platform not in data['package'][p.name].keys(): data['package'][p.name][p.platform] = p.version log.info('Adding new arch to package-config: ' '{}'.format(p.platform)) else: # Getting current version for platform value = data['package'][p.name][p.platform] # Updating version if applicable if p.version > value: log.info('Adding new version to package-config') data['package'][p.name][p.platform] = p.version return data def _cleanup(self, patch_manifest): # Remove old archives # were previously used to create patches if len(patch_manifest) < 1: return log.info('Cleaning up files directory') for p in patch_manifest: if os.path.exists(p['src']): basename = os.path.basename(p['src']) log.info('Removing {}'.format(basename)) os.remove(p['src']) def _make_patches(self, patch_manifest): pool_output = list() if len(patch_manifest) < 1: return pool_output log.info('Starting patch creation') try: cpu_count = multiprocessing.cpu_count() * 2 except Exception as err: log.debug(err, exc_info=True) log.warning('Cannot get cpu count from os. Using default 2') cpu_count = 2 pool = multiprocessing.Pool(processes=cpu_count) pool_output = pool.map(_make_patch, patch_manifest) return pool_output def _add_patches_to_packages(self, package_manifest, patches): # ToDo: Increase the efficiency of this double for # loop. Not sure if it can be done though if patches is not None and len(patches) >= 1: log.info('Adding patches to package list') for p in patches: # Not adding patch that are not complete if hasattr(p, 'ready') is False: continue # Not adding patch that are not complete if hasattr(p, 'ready') and p.ready is False: continue for pm in package_manifest: # if p.dst_filename == pm.filename: pm.patch_info['patch_name'] = \ os.path.basename(p.patch_name) # Don't try to get hash on a ghost file if not os.path.exists(p.patch_name): p_name = '' else: p_name = gph(p.patch_name) p_size = in_bytes(p.patch_name) pm.patch_info['patch_hash'] = p_name pm.patch_info['patch_size'] = p_size # No need to keep searching # We have the info we need for this patch break else: log.debug('No patch match found') else: if self.patch_support is True: log.warning('No patches found') return package_manifest def _update_version_file(self, json_data, package_manifest): # Updates version file with package meta-data log.info('Adding package meta-data to version manifest') easy_dict = EasyAccessDict(json_data) for p in package_manifest: patch_name = p.patch_info.get('patch_name') patch_hash = p.patch_info.get('patch_hash') patch_size = p.patch_info.get('patch_size') # Converting info to format compatible for version file # ToDo: Remove filename in version 2.0 info = {'file_hash': p.file_hash, 'file_size': p.file_size, 'file_name': p.filename, 'filename': p.filename} if patch_name and patch_hash: info['patch_name'] = patch_name info['patch_hash'] = patch_hash info['patch_size'] = patch_size version_key = '{}*{}*{}'.format(settings.UPDATES_KEY, p.name, p.version) version = easy_dict.get(version_key) log.debug('Package Info: {}'.format(version)) if version is None: log.debug('Adding new version to file') # First version this package name json_data[settings.UPDATES_KEY][p.name][p.version] = {} platform_key = '{}*{}*{}*{}'.format(settings.UPDATES_KEY, p.name, p.version, 'platform') platform = easy_dict.get(platform_key) if platform is None: name_ = json_data[settings.UPDATES_KEY][p.name] name_[p.version][p.platform] = info else: # package already present, adding another version to it log.debug('Appending info data to version file') # Used to keep within 80 characters n = p.name v = p.version json_data[settings.UPDATES_KEY][n][v][p.platform] = info # ToDo: Remove in future version. Maybe 2.0 # Backwards compat when implementing release channels if p.channel == 'stable': json_data['latest'][p.name][p.platform] = p.version # End ToDo # Add each package to latests section separated by release channel json_data['latest'][p.name][p.channel][p.platform] = p.version return json_data def _write_json_to_file(self, json_data): # Writes json data to disk log.debug('Saving version meta-data') self.db.save(settings.CONFIG_DB_KEY_VERSION_META, json_data) def _write_config_to_file(self, json_data): log.debug('Saving config data') self.db.save(settings.CONFIG_DB_KEY_PY_REPO_CONFIG, json_data) def _move_packages(self, package_manifest): if len(package_manifest) < 1: return log.info('Moving packages to deploy folder') for p in package_manifest: patch = p.patch_info.get('patch_name') with jms_utils.paths.ChDir(self.new_dir): if patch: if os.path.exists(os.path.join(self.deploy_dir, patch)): os.remove(os.path.join(self.deploy_dir, patch)) log.debug('Moving {} to {}'.format(patch, self.deploy_dir)) if os.path.exists(patch): shutil.move(patch, self.deploy_dir) shutil.copy(p.filename, self.deploy_dir) log.debug('Copying {} to {}'.format(p.filename, self.deploy_dir)) if os.path.exists(os.path.join(self.files_dir, p.filename)): os.remove(os.path.join(self.files_dir, p.filename)) shutil.move(p.filename, self.files_dir) log.debug('Moving {} to {}'.format(p.filename, self.files_dir)) def _update_file_list(self, json_data, package_info): files = json_data[settings.UPDATES_KEY] latest = json_data.get('latest') if latest is None: json_data['latest'] = {} file_name = files.get(package_info.name) if file_name is None: log.debug('Adding {} to file list'.format(package_info.name)) json_data[settings.UPDATES_KEY][package_info.name] = {} latest_package = json_data['latest'].get(package_info.name) if latest_package is None: json_data['latest'][package_info.name] = {} latest_channel = json_data['latest'][package_info.name].get('channel') if latest_channel is None: json_data['latest'][package_info.name][package_info.channel] = {} return json_data def _check_make_patch(self, json_data, name, platform): # Check to see if previous version is available to # make patch updates # Also calculates patch number log.info('Checking if patch creation is possible') if bsdiff4 is None: log.warning('Bsdiff is missing. Cannot create patches') return None src_file_path = None if os.path.exists(self.files_dir): with jms_utils.paths.ChDir(self.files_dir): files = os.listdir(os.getcwd()) files = remove_dot_files(files) # No src files to patch from. Exit quickly if len(files) == 0: return None # If latest not available in version file. Exit try: channel_dict = json_data['latest'][name]['channel'] latest = channel_dict[platform]['stable'] except KeyError: return None try: latest_platform = json_data[settings.UPDATES_KEY][name][latest] try: filename = latest_platform[platform]['file_name'] except KeyError: # ToDo: Remove in version 2.0 filename = latest_platform[platform]['filename'] # End ToDo except: return None src_file_path = os.path.join(self.files_dir, filename) try: patch_num = self.config['patches'][name] self.config['patches'][name] += 1 except KeyError: # If no patch number we will start at 1 patch_num = 1 if 'patches' not in self.config.keys(): self.config['patches'] = {} if name not in self.config['patches'].keys(): self.config['patches'][name] = patch_num + 1 num = patch_num + 1 log.debug('Patch Number: {}'.format(num)) return src_file_path, num return None
class ConfigManager(object): def __init__(self): self.cwd = os.getcwd() self.db = Storage() self.config_key = settings.CONFIG_DB_KEY_APP_CONFIG # Loads config from database (json file) def load_config(self): config_data = self.db.load(self.config_key) if config_data is None: config_data = {} config = Config() for k, v in config_data.items(): config[k] = v config.DATA_DIR = os.getcwd() return config def get_app_name(self): config = self.load_config() return config.APP_NAME # Saves config to database (json file) def save_config(self, obj): log.debug('Saving Config') self.db.save(self.config_key, obj) log.debug('Config saved') self.write_config_py(obj) log.debug('Wrote client config') # Writes client config to client_config.py def write_config_py(self, obj): keypack_data = self.db.load(settings.CONFIG_DB_KEY_KEYPACK) if keypack_data is None: log.debug('*** Keypack data is None ***') public_key = None else: public_key = keypack_data['client']['offline_public'] filename = os.path.join(self.cwd, *obj.CLIENT_CONFIG_PATH) attr_str_format = " {} = '{}'\n" attr_format = " {} = {}\n" log.debug('Writing client_config.py') with open(filename, 'w') as f: f.write('class ClientConfig(object):\n') log.debug('Adding PUBLIC_KEY to client_config.py') f.write(attr_str_format.format('PUBLIC_KEY', public_key)) if hasattr(obj, 'APP_NAME'): log.debug('Adding APP_NAME to client_config.py') f.write(attr_str_format.format('APP_NAME', obj.APP_NAME)) if hasattr(obj, 'COMPANY_NAME'): log.debug('Adding COMPANY_NAME to client_config.py') f.write(attr_str_format.format('COMPANY_NAME', obj.COMPANY_NAME)) if hasattr(obj, 'UPDATE_URLS'): log.debug('Adding UPDATE_URLS to client_config.py') f.write(attr_format.format('UPDATE_URLS', obj.UPDATE_URLS)) if hasattr(obj, 'MAX_DOWNLOAD_RETRIES'): log.debug('Adding MAX_DOWNLOAD_RETRIES to client_config.py') f.write(attr_format.format('MAX_DOWNLOAD_RETRIES', obj.MAX_DOWNLOAD_RETRIES))