def download_git_repos(): ''' Perform the inital checkout of the winrepo repositories. ''' try: import dulwich.client import dulwich.repo import dulwich.index except ImportError: raise CommandExecutionError('Command require dulwich python module') winrepo_dir = __salt__['config.get']('winrepo_dir') winrepo_remotes = __salt__['config.get']('winrepo_remotes') winrepo_remotes_ng = __salt__['config.get']('winrepo_remotes_ng') winrepo_dir_ng = __salt__['config.get']('winrepo_dir_ng') winrepo_cfg = [(winrepo_remotes, winrepo_dir), (winrepo_remotes_ng, winrepo_dir_ng)] ret = {} for remotes, base_dir in winrepo_cfg: for remote_info in remotes: try: if '/' in remote_info: targetname = remote_info.split('/')[-1] else: targetname = remote_info rev = 'HEAD' try: rev, remote_url = remote_info.strip().split() except ValueError: remote_url = remote_info gittarget = os.path.join(base_dir, targetname).replace('.', '_') client, path = dulwich.client.get_transport_and_path( remote_url) if os.path.exists(gittarget): local_repo = dulwich.repo.Repo(gittarget) else: os.makedirs(gittarget) local_repo = dulwich.repo.Repo.init(gittarget) remote_refs = client.fetch(remote_url, local_repo) if six.b(rev) not in remote_refs.refs: raise CommandExecutionError( 'Failed to find remote ref: {0}'.format(rev)) local_repo[six.b(rev)] = remote_refs[six.b(rev)] dulwich.index.build_index_from_tree( local_repo.path, local_repo.index_path(), local_repo.object_store, local_repo[six.b('head')].tree) ret.update({remote_info: True}) except Exception as exc: log.exception('Failed to process remote_info: %s', remote_info, exc_info=True) raise if __salt__['winrepo.genrepo'](): return ret return False
def __init__(self, pubdata): ''' Init an RSAX931Verifier instance :param str pubdata: The RSA public key in PEM format ''' pubdata = salt.utils.to_bytes(pubdata, 'ascii') pubdata = pubdata.replace(six.b('RSA '), six.b('')) self._bio = libcrypto.BIO_new_mem_buf(pubdata, len(pubdata)) self._rsa = c_void_p(libcrypto.RSA_new()) if not libcrypto.PEM_read_bio_RSA_PUBKEY(self._bio, pointer(self._rsa), None, None): raise ValueError('invalid RSA public key')
def test_key_management(self): """ Test key management """ do_key_name = self.instance_name + "-key" # generate key and fingerprint if salt.crypt.HAS_M2: rsa_key = salt.crypt.RSA.gen_key(4096, 65537, lambda: None) pub = six.b("ssh-rsa {}".format( base64.b64encode( six.b("\x00\x00\x00\x07ssh-rsa{}{}".format( *rsa_key.pub()))))) else: ssh_key = salt.crypt.RSA.generate(4096) pub = ssh_key.publickey().exportKey("OpenSSH") pub = salt.utils.stringutils.to_str(pub) key_hex = hashlib.md5(base64.b64decode( pub.strip().split()[1].encode())).hexdigest() finger_print = ":".join( [key_hex[x:x + 2] for x in range(0, len(key_hex), 2)]) try: _key = self.run_cloud( '-f create_key {0} name="{1}" public_key="{2}"'.format( self.PROVIDER, do_key_name, pub)) # Upload public key self.assertIn(finger_print, [i.strip() for i in _key]) # List all keys list_keypairs = self.run_cloud("-f list_keypairs {0}".format( self.PROVIDER)) self.assertIn(finger_print, [i.strip() for i in list_keypairs]) # List key show_keypair = self.run_cloud( "-f show_keypair {0} keyname={1}".format( self.PROVIDER, do_key_name)) self.assertIn(finger_print, [i.strip() for i in show_keypair]) except AssertionError: # Delete the public key if the above assertions fail self.run_cloud("-f remove_key {0} id={1}".format( self.PROVIDER, finger_print)) raise finally: # Delete public key self.assertTrue( self.run_cloud("-f remove_key {0} id={1}".format( self.PROVIDER, finger_print)))
def test_master_startup(self): proc = NonBlockingPopen( [ self.get_script_path("master"), "-c", RUNTIME_VARS.TMP_CONF_DIR, "-l", "info", ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) out = six.b("") err = six.b("") # Testing this should never be longer than 1 minute max_time = time.time() + 60 try: while True: if time.time() > max_time: assert False, "Max timeout ocurred" time.sleep(0.5) _out = proc.recv() _err = proc.recv_err() if _out: out += _out if _err: err += _err if six.b("DeprecationWarning: object() takes no parameters" ) in out: self.fail( "'DeprecationWarning: object() takes no parameters' was seen in output" ) if six.b("TypeError: object() takes no parameters") in out: self.fail( "'TypeError: object() takes no parameters' was seen in output" ) if six.b("Setting up the master communication server") in out: # We got past the place we need, stop the process break if out is None and err is None: break if proc.poll() is not None: break finally: terminate_process(proc.pid, kill_children=True)
def test_sign_message_with_passphrase(self): key = M2Crypto.RSA.load_key_string(six.b(PRIVKEY_DATA)) with patch("salt.crypt.get_rsa_key", return_value=key): self.assertEqual( SIG, crypt.sign_message("/keydir/keyname.pem", MSG, passphrase="password"), )
def _check_cmdline(data): ''' In some cases where there are an insane number of processes being created on a system a PID can get recycled or assigned to a non-Salt process. On Linux this fn checks to make sure the PID we are checking on is actually a Salt process. For non-Linux systems we punt and just return True ''' if not salt.utils.is_linux(): return True pid = data.get('pid') if not pid: return False if not os.path.isdir('/proc'): return True path = os.path.join('/proc/{0}/cmdline'.format(pid)) if not os.path.isfile(path): return False try: with salt.utils.files.fopen(path, 'rb') as fp_: if six.b('salt') in fp_.read(): return True except (OSError, IOError): return False
def setUpClass(cls): ret_port = get_unused_localhost_port() publish_port = get_unused_localhost_port() tcp_master_pub_port = get_unused_localhost_port() tcp_master_pull_port = get_unused_localhost_port() tcp_master_publish_pull = get_unused_localhost_port() tcp_master_workers = get_unused_localhost_port() cls.master_config = cls.get_temp_config( 'master', **{'transport': 'zeromq', 'auto_accept': True, 'ret_port': ret_port, 'publish_port': publish_port, 'tcp_master_pub_port': tcp_master_pub_port, 'tcp_master_pull_port': tcp_master_pull_port, 'tcp_master_publish_pull': tcp_master_publish_pull, 'tcp_master_workers': tcp_master_workers, 'sign_pub_messages': False, } ) salt.master.SMaster.secrets['aes'] = { 'secret': multiprocessing.Array( ctypes.c_char, six.b(salt.crypt.Crypticle.generate_key_string()), ), } cls.minion_config = cls.get_temp_config( 'minion', **{'transport': 'zeromq', 'master_ip': '127.0.0.1', 'master_port': ret_port, 'auth_timeout': 5, 'auth_tries': 1, 'master_uri': 'tcp://127.0.0.1:{0}'.format(ret_port)} )
def setUpClass(cls): ret_port = get_unused_localhost_port() publish_port = get_unused_localhost_port() tcp_master_pub_port = get_unused_localhost_port() tcp_master_pull_port = get_unused_localhost_port() tcp_master_publish_pull = get_unused_localhost_port() tcp_master_workers = get_unused_localhost_port() cls.master_config = cls.get_temp_config( "master", **{ "transport": "zeromq", "auto_accept": True, "ret_port": ret_port, "publish_port": publish_port, "tcp_master_pub_port": tcp_master_pub_port, "tcp_master_pull_port": tcp_master_pull_port, "tcp_master_publish_pull": tcp_master_publish_pull, "tcp_master_workers": tcp_master_workers, "sign_pub_messages": False, }) salt.master.SMaster.secrets["aes"] = { "secret": multiprocessing.Array( ctypes.c_char, six.b(salt.crypt.Crypticle.generate_key_string()), ), } cls.minion_config = cls.get_temp_config( "minion", **{ "transport": "zeromq", "master_ip": "127.0.0.1", "master_port": ret_port, "auth_timeout": 5, "auth_tries": 1, "master_uri": "tcp://127.0.0.1:{0}".format(ret_port), })
def setUpClass(cls): try: os.makedirs(GPG_HOMEDIR, mode=0o700) except Exception: cls.created_gpg_homedir = False raise else: cls.created_gpg_homedir = True cmd_prefix = ['gpg', '--homedir', GPG_HOMEDIR] cmd = cmd_prefix + ['--list-keys'] log.debug('Instantiating gpg keyring using: %s', cmd) output = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=False).communicate()[0] log.debug('Result:\n%s', output) cmd = cmd_prefix + ['--import', '--allow-secret-key-import'] log.debug('Importing keypair using: %s', cmd) output = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=False).communicate(input=six.b(TEST_KEY))[0] log.debug('Result:\n%s', output) os.makedirs(PILLAR_BASE) with salt.utils.fopen(TOP_SLS, 'w') as fp_: fp_.write( textwrap.dedent('''\ base: '*': - gpg ''')) with salt.utils.fopen(GPG_SLS, 'w') as fp_: fp_.write(GPG_PILLAR_YAML)
def test_master_startup(self): proc = NonBlockingPopen( [ self.get_script_path('master'), '-c', self.config_dir, '-l', 'info' ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) out = six.b('') err = six.b('') # Testing this should never be longer than 1 minute max_time = time.time() + 60 try: while True: if time.time() > max_time: assert False, 'Max timeout ocurred' time.sleep(0.5) _out = proc.recv() _err = proc.recv_err() if _out: out += _out if _err: err += _err if six.b('DeprecationWarning: object() takes no parameters') in out: self.fail('\'DeprecationWarning: object() takes no parameters\' was seen in output') if six.b('TypeError: object() takes no parameters') in out: self.fail('\'TypeError: object() takes no parameters\' was seen in output') if six.b('Setting up the master communication server') in out: # We got past the place we need, stop the process break if out is None and err is None: break if proc.poll() is not None: break finally: terminate_process(proc.pid, kill_children=True)
def test_out_token_defined(self): mock_context = MagicMock(return_value=MagicMock()) mock_context.return_value.established = False mock_context.return_value.step = MagicMock(return_value='out_token') with patch.object(salt.utils.vmware.gssapi, 'InitContext', mock_context): ret = salt.utils.vmware.get_gssapi_token('principal', 'host', 'domain') self.assertEqual(mock_context.return_value.step.called, 1) self.assertEqual(ret, base64.b64encode(six.b('out_token')))
def test_sdecode(self): b = six.b('\xe7\xb9\x81\xe4\xbd\x93') u = u'\u7e41\u4f53' if six.PY2: # Under Py3, the above `b` as bytes, will never decode as anything even comparable using `ascii` # but no unicode error will be raised, as such, sdecode will return the poorly decoded string with patch('salt.utils.locales.get_encodings', return_value=['ascii']): self.assertEqual(locales.sdecode(b), b) # no decode with patch('salt.utils.locales.get_encodings', return_value=['utf-8']): self.assertEqual(locales.sdecode(b), u)
def test_pip_installed_specific_env(self): # Create the testing virtualenv venv_dir = os.path.join( RUNTIME_VARS.TMP, 'pip-installed-specific-env' ) # Let's write a requirements file requirements_file = os.path.join( RUNTIME_VARS.TMP_PRODENV_STATE_TREE, 'prod-env-requirements.txt' ) with salt.utils.files.fopen(requirements_file, 'wb') as reqf: reqf.write(six.b('pep8\n')) try: self.run_function('virtualenv.create', [venv_dir]) # The requirements file should not be found the base environment ret = self.run_state( 'pip.installed', name='', bin_env=venv_dir, requirements='salt://prod-env-requirements.txt' ) self.assertSaltFalseReturn(ret) self.assertInSaltComment( "'salt://prod-env-requirements.txt' not found", ret ) # The requirements file must be found in the prod environment ret = self.run_state( 'pip.installed', name='', bin_env=venv_dir, saltenv='prod', requirements='salt://prod-env-requirements.txt' ) self.assertSaltTrueReturn(ret) self.assertInSaltComment( 'Successfully processed requirements file ' 'salt://prod-env-requirements.txt', ret ) # We're using the base environment but we're passing the prod # environment as an url arg to salt:// ret = self.run_state( 'pip.installed', name='', bin_env=venv_dir, requirements='salt://prod-env-requirements.txt?saltenv=prod' ) self.assertSaltTrueReturn(ret) self.assertInSaltComment( 'Requirements were already installed.', ret ) finally: if os.path.isdir(venv_dir): shutil.rmtree(venv_dir) if os.path.isfile(requirements_file): os.unlink(requirements_file)
def test_store_success(self): ''' Tests that the store function writes the data to the serializer for storage. ''' # Create a temporary cache dir tmp_dir = tempfile.mkdtemp(dir=integration.SYS_TMP_DIR) # Use the helper function to create the cache file using localfs.store() self._create_tmp_cache_file(tmp_dir, salt.payload.Serial(self)) # Read in the contents of the key.p file and assert "payload data" was written with salt.utils.fopen(tmp_dir + '/bank/key.p', 'rb') as fh_: for line in fh_: self.assertIn(six.b('payload data'), line)
def tearDownClass(cls): cmd = ['gpg-connect-agent', '--homedir', GPG_HOMEDIR] try: log.debug('Killing gpg-agent using: %s', cmd) output = subprocess.Popen( cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False).communicate(input=six.b('KILLAGENT'))[0] log.debug('Result:\n%s', output) except OSError: log.debug('No need to kill: old gnupg doesn\'t start the agent.') if cls.created_gpg_homedir: try: shutil.rmtree(GPG_HOMEDIR) except OSError as exc: # GPG socket can disappear before rmtree gets to this point if exc.errno != errno.ENOENT: raise shutil.rmtree(PILLAR_BASE)
def _iterate_read_data(read_data): # Helper for mock_open: # Retrieve lines from read_data via a generator so that separate calls to # readline, read, and readlines are properly interleaved if six.PY3 and isinstance(read_data, six.binary_type): data_as_list = [ '{0}\n'.format(l.decode(__salt_system_encoding__)) for l in read_data.split(six.b('\n')) ] else: data_as_list = ['{0}\n'.format(l) for l in read_data.split('\n')] if data_as_list[-1] == '\n': # If the last line ended in a newline, the list comprehension will have an # extra entry that's just a newline. Remove this. data_as_list = data_as_list[:-1] else: # If there wasn't an extra newline by itself, then the file being # emulated doesn't have a newline to end the last line remove the # newline that our naive format() added data_as_list[-1] = data_as_list[-1][:-1] for line in data_as_list: yield line
def get_url(self, url, dest, makedirs=False, saltenv='base', no_cache=False, cachedir=None): ''' Get a single file from a URL. ''' url_data = urlparse(url) url_scheme = url_data.scheme url_path = os.path.join( url_data.netloc, url_data.path).rstrip(os.sep) if url_scheme and url_scheme.lower() in string.ascii_lowercase: url_path = ':'.join((url_scheme, url_path)) url_scheme = 'file' if url_scheme in ('file', ''): # Local filesystem if not os.path.isabs(url_path): raise CommandExecutionError( 'Path \'{0}\' is not absolute'.format(url_path) ) if dest is None: with salt.utils.fopen(url_path, 'r') as fp_: data = fp_.read() return data return url_path if url_scheme == 'salt': result = self.get_file(url, dest, makedirs, saltenv, cachedir=cachedir) if result and dest is None: with salt.utils.fopen(result, 'r') as fp_: data = fp_.read() return data return result if dest: destdir = os.path.dirname(dest) if not os.path.isdir(destdir): if makedirs: os.makedirs(destdir) else: return '' elif not no_cache: dest = self._extrn_path(url, saltenv, cachedir=cachedir) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) if url_data.scheme == 's3': try: def s3_opt(key, default=None): '''Get value of s3.<key> from Minion config or from Pillar''' if 's3.' + key in self.opts: return self.opts['s3.' + key] try: return self.opts['pillar']['s3'][key] except (KeyError, TypeError): return default self.utils['s3.query'](method='GET', bucket=url_data.netloc, path=url_data.path[1:], return_bin=False, local_file=dest, action=None, key=s3_opt('key'), keyid=s3_opt('keyid'), service_url=s3_opt('service_url'), verify_ssl=s3_opt('verify_ssl', True), location=s3_opt('location')) return dest except Exception as exc: raise MinionError( 'Could not fetch from {0}. Exception: {1}'.format(url, exc) ) if url_data.scheme == 'ftp': try: ftp = ftplib.FTP() ftp.connect(url_data.hostname, url_data.port) ftp.login(url_data.username, url_data.password) with salt.utils.fopen(dest, 'wb') as fp_: ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write) ftp.quit() return dest except Exception as exc: raise MinionError('Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc)) if url_data.scheme == 'swift': try: def swift_opt(key, default): '''Get value of <key> from Minion config or from Pillar''' if key in self.opts: return self.opts[key] try: return self.opts['pillar'][key] except (KeyError, TypeError): return default swift_conn = SaltSwift(swift_opt('keystone.user', None), swift_opt('keystone.tenant', None), swift_opt('keystone.auth_url', None), swift_opt('keystone.password', None)) swift_conn.get_object(url_data.netloc, url_data.path[1:], dest) return dest except Exception: raise MinionError('Could not fetch from {0}'.format(url)) get_kwargs = {} if url_data.username is not None \ and url_data.scheme in ('http', 'https'): netloc = url_data.netloc at_sign_pos = netloc.rfind('@') if at_sign_pos != -1: netloc = netloc[at_sign_pos + 1:] fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) get_kwargs['auth'] = (url_data.username, url_data.password) else: fixed_url = url destfp = None try: # Tornado calls streaming_callback on redirect response bodies. # But we need streaming to support fetching large files (> RAM avail). # Here we working this around by disabling recording the body for redirections. # The issue is fixed in Tornado 4.3.0 so on_header callback could be removed # when we'll deprecate Tornado<4.3.0. # See #27093 and #30431 for details. # Use list here to make it writable inside the on_header callback. Simple bool doesn't # work here: on_header creates a new local variable instead. This could be avoided in # Py3 with 'nonlocal' statement. There is no Py2 alternative for this. write_body = [None, False, None] def on_header(hdr): if write_body[1] is not False and write_body[2] is None: # Try to find out what content type encoding is used if this is a text file write_body[1].parse_line(hdr) # pylint: disable=no-member if 'Content-Type' in write_body[1]: content_type = write_body[1].get('Content-Type') # pylint: disable=no-member if not content_type.startswith('text'): write_body[1] = write_body[2] = False else: encoding = 'utf-8' fields = content_type.split(';') for field in fields: if 'encoding' in field: encoding = field.split('encoding=')[-1] write_body[2] = encoding # We have found our encoding. Stop processing headers. write_body[1] = False if write_body[0] is not None: # We already parsed the first line. No need to run the code below again return try: hdr = parse_response_start_line(hdr) except HTTPInputError: # Not the first line, do nothing return write_body[0] = hdr.code not in [301, 302, 303, 307] write_body[1] = HTTPHeaders() if no_cache: result = [] def on_chunk(chunk): if write_body[0]: if write_body[2]: chunk = chunk.decode(write_body[2]) result.append(chunk) else: dest_tmp = "{0}.part".format(dest) # We need an open filehandle to use in the on_chunk callback, # that's why we're not using a with clause here. destfp = salt.utils.fopen(dest_tmp, 'wb') def on_chunk(chunk): if write_body[0]: destfp.write(chunk) query = salt.utils.http.query( fixed_url, stream=True, streaming_callback=on_chunk, header_callback=on_header, username=url_data.username, password=url_data.password, opts=self.opts, **get_kwargs ) if 'handle' not in query: raise MinionError('Error: {0} reading {1}'.format(query['error'], url)) if no_cache: if write_body[2]: return six.u('').join(result) return six.b('').join(result) else: destfp.close() destfp = None salt.utils.files.rename(dest_tmp, dest) return dest except HTTPError as exc: raise MinionError('HTTP error {0} reading {1}: {3}'.format( exc.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code])) except URLError as exc: raise MinionError('Error reading {0}: {1}'.format(url, exc.reason)) finally: if destfp is not None: destfp.close()
def __init__(self, opts, **kwargs): self.opts = opts self.ttype = 'zeromq' self.io_loop = kwargs.get('io_loop') if self.io_loop is None: zmq.eventloop.ioloop.install() self.io_loop = tornado.ioloop.IOLoop.current() self.hexid = hashlib.sha1(six.b(self.opts['id'])).hexdigest() self.auth = salt.crypt.AsyncAuth(self.opts, io_loop=self.io_loop) self.serial = salt.payload.Serial(self.opts) self.context = zmq.Context() self._socket = self.context.socket(zmq.SUB) if self.opts['zmq_filtering']: # TODO: constants file for "broadcast" self._socket.setsockopt(zmq.SUBSCRIBE, 'broadcast') self._socket.setsockopt(zmq.SUBSCRIBE, self.hexid) else: self._socket.setsockopt(zmq.SUBSCRIBE, '') self._socket.setsockopt(zmq.SUBSCRIBE, '') self._socket.setsockopt(zmq.IDENTITY, self.opts['id']) # TODO: cleanup all the socket opts stuff if hasattr(zmq, 'TCP_KEEPALIVE'): self._socket.setsockopt( zmq.TCP_KEEPALIVE, self.opts['tcp_keepalive'] ) self._socket.setsockopt( zmq.TCP_KEEPALIVE_IDLE, self.opts['tcp_keepalive_idle'] ) self._socket.setsockopt( zmq.TCP_KEEPALIVE_CNT, self.opts['tcp_keepalive_cnt'] ) self._socket.setsockopt( zmq.TCP_KEEPALIVE_INTVL, self.opts['tcp_keepalive_intvl'] ) recon_delay = self.opts['recon_default'] if self.opts['recon_randomize']: recon_delay = randint(self.opts['recon_default'], self.opts['recon_default'] + self.opts['recon_max'] ) log.debug("Generated random reconnect delay between '{0}ms' and '{1}ms' ({2})".format( self.opts['recon_default'], self.opts['recon_default'] + self.opts['recon_max'], recon_delay) ) log.debug("Setting zmq_reconnect_ivl to '{0}ms'".format(recon_delay)) self._socket.setsockopt(zmq.RECONNECT_IVL, recon_delay) if hasattr(zmq, 'RECONNECT_IVL_MAX'): log.debug("Setting zmq_reconnect_ivl_max to '{0}ms'".format( self.opts['recon_default'] + self.opts['recon_max']) ) self._socket.setsockopt( zmq.RECONNECT_IVL_MAX, self.opts['recon_max'] ) if self.opts['ipv6'] is True and hasattr(zmq, 'IPV4ONLY'): # IPv6 sockets work for both IPv6 and IPv4 addresses self._socket.setsockopt(zmq.IPV4ONLY, 0) if HAS_ZMQ_MONITOR and self.opts['zmq_monitor']: self._monitor = ZeroMQSocketMonitor(self._socket) self._monitor.start_io_loop(self.io_loop)