def test_unix_server(self):
sockpath = os.path.join(os.getcwd(),
'test.{0}.sock'.format(os.getpid()))
sockurl = 'unix://' + sockpath
context = RpcContext()
context.register_service('test', TestService)
server = Server()
server.rpc = context
server.start(sockurl)
threading.Thread(target=server.serve_forever, daemon=True).start()
# Spin until server is ready
while not os.path.exists(sockpath):
time.sleep(0.1)
client = Client()
client.connect(sockurl)
self.assertTrue(client.connected)
self.assertEqual(client.call_sync('test.hello', 'freenas'),
'Hello World, freenas')
client.disconnect()
server.close()
os.unlink(sockpath)
def init_dispatcher(self):
def on_error(reason, **kwargs):
if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
self.logger.warning('Connection to dispatcher lost')
self.connect()
self.client = Client()
self.client.on_error(on_error)
self.connect()
def setUp(self):
super(BaseTestCase, self).setUp()
assert self.context is not None
self.ssh_client = self.context.ssh_client
self.client = Client()
self.client.connect('ws://{0}'.format(self.context.hostname))
self.client.login_user(self.context.username, self.context.password)
load_schema_definitions(self.client)
def main(name, *args):
connection = Client()
connection.connect('127.0.0.1')
connection.login_service('ups')
connection.emit_event('service.ups.signal', {
'name': name,
'type': os.environ['NOTIFYTYPE'],
})
connection.disconnect()
def setUp(self):
try:
self.conn = Client()
self.conn.event_callback = self.on_event
self.conn.connect(os.getenv('TESTHOST', '127.0.0.1'))
self.conn.login_user(os.getenv('TESTUSER', 'root'),
os.getenv('TESTPWD', ''),
timeout=self.task_timeout)
self.conn.subscribe_events('*')
except:
raise
def main(*args):
connection = Client()
connection.connect('127.0.0.1')
connection.login_service('smtp')
parser = argparse.ArgumentParser(description='Process email')
parser.add_argument('-i', dest='strip_leading_dot', action='store_false',
default=True, help='see sendmail(8) -i')
parser.add_argument('-t', dest='parse_recipients', action='store_true',
default=False,
help='parse recipients from message')
parser.usage = ' '.join(parser.format_usage().split(' ')[1:-1])
parser.usage += ' [email_addr|user] ..'
args, to_addrs = parser.parse_known_args()
if not to_addrs and not args.parse_recipients:
parser.exit(message=parser.format_usage())
msg = sys.stdin.read()
em_parser = email.parser.Parser()
em = em_parser.parsestr(msg)
if args.parse_recipients:
# Strip away the comma based delimiters and whitespace.
to_addrs = map(str.strip, em.get('To').split(','))
if not to_addrs or not to_addrs[0]:
to_addrs = ['root']
margs = {}
margs['extra_headers'] = dict(em)
margs['extra_headers'].update({
'X-Mailer': 'FreeNAS',
'X-FreeNAS-Host': socket.gethostname(),
})
margs['subject'] = em.get('Subject')
if em.is_multipart():
margs['attachments'] = filter(
lambda part: part.get_content_maintype() != 'multipart',
em.walk()
)
margs['message'] = (
'This is a MIME formatted message. If you see '
'this text it means that your email software '
'does not support MIME formatted messages.')
else:
margs['message'] = ''.join(email.iterators.body_line_iterator(em))
if to_addrs:
margs['to'] = to_addrs
connection.call_sync('mail.send', margs)
connection.disconnect()
def test_back_to_back(self):
a, b = socket.socketpair()
self.assertGreaterEqual(a.fileno(), 0)
self.assertGreaterEqual(b.fileno(), 0)
c1 = Client()
c1.standalone_server = True
c1.enable_server()
c1.register_service('test', TestService())
c1.connect('fd://{0}'.format(a.fileno()))
self.assertTrue(c1.connected)
c2 = Client()
c2.connect('fd://{0}'.format(b.fileno()))
self.assertTrue(c2.connected)
self.assertEqual(c2.call_sync('test.hello', 'freenas'),
'Hello World, freenas')
c2.disconnect()
a.close()
c1.disconnect()
b.close()
def setup_back_to_back(self, streaming=False):
a, b = socket.socketpair()
self.assertGreaterEqual(a.fileno(), 0)
self.assertGreaterEqual(b.fileno(), 0)
c1 = Client()
c1._s = a
c1.enable_server()
c1.standalone_server = True
if streaming:
c1.streaming = True
c1.rpc.streaming_enabled = True
c1.register_service('test', TestService())
c1.connect('fd://{0}'.format(a.fileno()))
self.assertTrue(c1.connected)
c2 = Client()
c2._s = b
c2.streaming = True
c2.connect('fd://{0}'.format(b.fileno()))
self.assertTrue(c2.connected)
return c1, c2
def __init__(self):
self.logger = logging.getLogger(self.__class__.__name__)
self.msock = msock.client.Client()
self.msock.on_closed = self.on_msock_close
self.rpc_fd = -1
self.connection_id = None
self.jobs = []
self.state = ConnectionState.OFFLINE
self.config = None
self.keepalive = None
self.connected_at = None
self.cv = Condition()
self.rpc = RpcContext()
self.client = Client()
self.server = Server()
self.middleware_endpoint = None
def get_freenas_peer_client(parent, remote):
try:
address = socket.gethostbyname(remote)
except socket.error as err:
raise TaskException(err.errno, '{0} is unreachable'.format(remote))
host = parent.dispatcher.call_sync(
'peer.query', [
('or', [
('credentials.address', '=', remote),
('credentials.address', '=', address),
]),
('type', '=', 'freenas')
],
{'single': True}
)
if not host:
raise TaskException(errno.ENOENT, 'There are no known keys to connect to {0}'.format(remote))
with io.StringIO() as f:
f.write(parent.configstore.get('peer.freenas.key.private'))
f.seek(0)
pkey = RSAKey.from_private_key(f)
credentials = host['credentials']
try:
client = Client()
with tempfile.NamedTemporaryFile('w') as host_key_file:
host_key_file.write(remote + ' ' + credentials['hostkey'])
host_key_file.flush()
client.connect(
'ws+ssh://freenas@{0}'.format(wrap_address(remote)),
port=credentials['port'],
host_key_file=host_key_file.name,
pkey=pkey
)
client.login_service('replicator')
return client
except (AuthenticationException, SSHException):
raise TaskException(errno.EAUTH, 'Cannot connect to {0}'.format(remote))
except OSError as err:
raise TaskException(errno.ECONNREFUSED, 'Cannot connect to {0}: {1}'.format(remote, err))
def connect_keepalive(self):
while True:
try:
if not self.connection_id:
self.connection_id = uuid.uuid4()
self.msock.connect(SUPPORT_PROXY_ADDRESS)
self.logger.info(
'Connecting to {0}'.format(SUPPORT_PROXY_ADDRESS))
self.rpc_fd = self.msock.create_channel(0)
time.sleep(1) # FIXME
self.client = Client()
self.client.connect('fd://', fobj=self.rpc_fd)
self.client.channel_serializer = MSockChannelSerializer(
self.msock)
self.client.standalone_server = True
self.client.enable_server()
self.client.register_service('debug', DebugService(self))
self.client.call_sync('server.login', str(self.connection_id),
socket.gethostname(), get_version(),
'none')
self.set_state(ConnectionState.CONNECTED)
except BaseException as err:
self.logger.warning(
'Failed to initiate support connection: {0}'.format(err),
exc_info=True)
self.msock.disconnect()
else:
self.connected_at = datetime.now()
with self.cv:
self.cv.wait_for(lambda: self.state in (
ConnectionState.LOST, ConnectionState.OFFLINE))
if self.state == ConnectionState.OFFLINE:
return
self.logger.warning(
'Support connection lost, retrying in 10 seconds')
time.sleep(10)
def process_request(self, req, resp):
# Do not require auth to access index
if req.relative_uri == '/':
return
auth = req.get_header("Authorization")
if auth is None or not auth.startswith('Basic '):
raise falcon.HTTPUnauthorized(
'Authorization token required',
'Provide a Basic Authentication header',
['Basic realm="FreeNAS"'],
)
try:
username, password = base64.b64decode(
auth[6:]).decode('utf8').split(':', 1)
except binascii.Error:
raise falcon.HTTPUnauthorized(
'Invalid Authorization token',
'Provide a valid Basic Authentication header',
['Basic realm="FreeNAS"'],
)
try:
client = Client()
client.connect('unix:')
client.login_user(username, password, check_password=True)
req.context['client'] = client
except RpcException as e:
if e.code == errno.EACCES:
raise falcon.HTTPUnauthorized(
'Invalid credentials',
'Verify your credentials and try again.',
['Basic realm="FreeNAS"'],
)
raise falcon.HTTPUnauthorized('Unknown authentication error',
str(e), ['Basic realm="FreeNAS"'])
def init_dispatcher(self):
self.client = Client()
self.client.on_error(self.connection_error)
self.connect()
def main(self):
if len(sys.argv) != 2:
print("Invalid number of arguments", file=sys.stderr)
sys.exit(errno.EINVAL)
key = sys.argv[1]
configure_logging(None, logging.DEBUG)
self.datastore = get_datastore()
self.configstore = ConfigStore(self.datastore)
self.conn = Client()
self.conn.connect('unix:')
self.conn.login_service('task.{0}'.format(os.getpid()))
self.conn.enable_server()
self.conn.call_sync('management.enable_features',
['streaming_responses'])
self.conn.rpc.register_service_instance('taskproxy', self.service)
self.conn.register_event_handler('task.progress',
self.task_progress_handler)
self.conn.call_sync('task.checkin', key)
setproctitle.setproctitle('task executor (idle)')
while True:
try:
task = self.task.get()
logging.root.setLevel(
self.conn.call_sync('management.get_logging_level'))
setproctitle.setproctitle('task executor (tid {0})'.format(
task['id']))
if task['debugger']:
sys.path.append('/usr/local/lib/dispatcher/pydev')
import pydevd
host, port = task['debugger']
pydevd.settrace(host,
port=port,
stdoutToServer=True,
stderrToServer=True)
name, _ = os.path.splitext(os.path.basename(task['filename']))
module = self.module_cache.get(task['filename'])
if not module:
module = load_module_from_file(name, task['filename'])
self.module_cache[task['filename']] = module
setproctitle.setproctitle('task executor (tid {0})'.format(
task['id']))
fds = list(self.collect_fds(task['args']))
try:
dispatcher = DispatcherWrapper(self.conn)
self.instance = getattr(module,
task['class'])(dispatcher,
self.datastore)
self.instance.configstore = self.configstore
self.instance.user = task['user']
self.instance.environment = task['environment']
self.running.set()
self.run_task_hooks(self.instance, task, 'before')
result = self.instance.run(*task['args'])
self.run_task_hooks(self.instance,
task,
'after',
result=result)
except BaseException as err:
print("Task exception: {0}".format(str(err)),
file=sys.stderr)
traceback.print_exc(file=sys.stderr)
if hasattr(self.instance, 'rollback'):
self.put_status('ROLLBACK')
try:
self.instance.rollback(*task['args'])
except BaseException as rerr:
print("Task exception during rollback: {0}".format(
str(rerr)),
file=sys.stderr)
traceback.print_exc(file=sys.stderr)
# Main task is already failed at this point, so ignore hook errors
with contextlib.suppress(RpcException):
self.run_task_hooks(self.instance,
task,
'error',
error=serialize_error(err))
self.put_status('FAILED', exception=err)
else:
self.put_status('FINISHED', result=result)
finally:
self.close_fds(fds)
self.running.clear()
except RpcException as err:
print("RPC failed: {0}".format(str(err)), file=sys.stderr)
print(traceback.format_exc(), flush=True)
sys.exit(errno.EBADMSG)
except socket.error as err:
print("Cannot connect to dispatcher: {0}".format(str(err)),
file=sys.stderr)
sys.exit(errno.ETIMEDOUT)
if task['debugger']:
import pydevd
pydevd.stoptrace()
setproctitle.setproctitle('task executor (idle)')
def run(self, peer, initial_credentials):
hostid = self.dispatcher.call_sync('system.info.host_uuid')
hostname = self.dispatcher.call_sync(
'system.general.get_config')['hostname']
remote_peer_name = hostname
credentials = peer['credentials']
remote = credentials.get('address')
port = credentials.get('port', 22)
username = initial_credentials.get('username')
password = initial_credentials.get('password')
auth_code = initial_credentials.get('auth_code')
key_auth = initial_credentials.get('key_auth')
local_ssh_config = self.dispatcher.call_sync('service.sshd.get_config')
if self.datastore.exists('peers', ('credentials.address', '=', remote),
('type', '=', 'freenas')):
raise TaskException(
errno.EEXIST,
'FreeNAS peer entry for {0} already exists'.format(remote))
remote_client = Client()
try:
if auth_code:
try:
remote_client.connect('ws://{0}'.format(
wrap_address(remote)))
except (AuthenticationException, OSError,
ConnectionRefusedError):
raise TaskException(
errno.ECONNABORTED,
'Cannot connect to {0}:{1}'.format(remote, port))
try:
remote_host_uuid, pubkey = remote_client.call_sync(
'peer.freenas.auth_with_code', auth_code, hostname,
local_ssh_config['port'])
except RpcException as err:
raise TaskException(err.code, err.message)
try:
self.dispatcher.call_sync('peer.freenas.put_temp_pubkey',
pubkey)
if not self.dispatcher.test_or_wait_for_event(
'peer.changed',
lambda ar: ar['operation'] == 'create' and
remote_host_uuid in ar['ids'],
lambda: self.datastore.exists(
'peers', ('id', '=', remote_host_uuid)),
timeout=30):
raise TaskException(
errno.EAUTH,
'FreeNAS peer creation failed. Check connection to host {0}.'
.format(remote))
finally:
self.dispatcher.call_sync(
'peer.freenas.remove_temp_pubkey', pubkey)
else:
try:
if key_auth:
with io.StringIO() as f:
f.write(
self.configstore.get(
'peer.freenas.key.private'))
f.seek(0)
pkey = RSAKey.from_private_key(f)
max_tries = 50
while True:
try:
remote_client.connect(
'ws+ssh://freenas@{0}'.format(
wrap_address(remote)),
pkey=pkey,
port=port)
break
except AuthenticationException:
if max_tries:
max_tries -= 1
time.sleep(1)
else:
raise
else:
remote_client.connect('ws+ssh://{0}@{1}'.format(
username, wrap_address(remote)),
port=port,
password=password)
remote_client.login_service('replicator')
except (AuthenticationException, OSError,
ConnectionRefusedError):
raise TaskException(
errno.ECONNABORTED,
'Cannot connect to {0}:{1}'.format(remote, port))
local_host_key, local_pub_key = self.dispatcher.call_sync(
'peer.freenas.get_ssh_keys')
remote_host_key, remote_pub_key = remote_client.call_sync(
'peer.freenas.get_ssh_keys')
ip_at_remote_side = remote_client.local_address[0]
remote_hostname = remote_client.call_sync(
'system.general.get_config')['hostname']
remote_host_key = remote_host_key.rsplit(' ', 1)[0]
local_host_key = local_host_key.rsplit(' ', 1)[0]
if remote_client.call_sync('peer.query',
[('id', '=', hostid)]):
raise TaskException(
errno.EEXIST,
'Peer entry of {0} already exists at {1}'.format(
hostname, remote))
peer['credentials'] = {
'%type': 'freenas-credentials',
'pubkey': remote_pub_key,
'hostkey': remote_host_key,
'port': port,
'address': remote_hostname
}
local_id = remote_client.call_sync('system.info.host_uuid')
peer['id'] = local_id
peer['name'] = remote_hostname
ip = socket.gethostbyname(remote)
created_ids = self.join_subtasks(
self.run_subtask('peer.freenas.create_local', peer, ip,
True))
peer['id'] = hostid
peer['name'] = remote_peer_name
peer['credentials'] = {
'%type': 'freenas-credentials',
'pubkey': local_pub_key,
'hostkey': local_host_key,
'port': local_ssh_config['port'],
'address': hostname
}
try:
call_task_and_check_state(remote_client,
'peer.freenas.create_local',
peer, ip_at_remote_side)
except TaskException:
self.datastore.delete('peers', local_id)
self.dispatcher.dispatch_event('peer.changed', {
'operation': 'delete',
'ids': [local_id]
})
raise
return created_ids[0]
finally:
remote_client.disconnect()
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
#####################################################################
from threading import Lock
from freenas.dispatcher.client import Client
from freenas.dispatcher.rpc import RpcException
SERVICED_SOCKET = 'unix:///var/run/serviced.sock'
_client = Client()
_lock = Lock()
class ServicedException(RpcException):
pass
def checkin():
with _lock:
try:
_client.connect(SERVICED_SOCKET)
return _client.call_sync('serviced.job.checkin')
except RpcException as err:
raise ServicedException(err.code, err.message, err.extra)
finally:
def __init__(self, uri, type=None, username='******', password=''):
super(FileWrapRemote, self).__init__(uri, type)
self.client = Client()
self.username = username
self.password = password
def __init__(self, level=logging.NOTSET, address=None, ident=None):
super(LogdLogHandler, self).__init__(level)
self.address = address or 'unix:///var/run/logd.sock'
self.ident = ident or os.path.basename(sys.executable)
self.client = Client()
self.client.connect(self.address)
def initialize(self, context):
self.client = Client()
self.client.connect('unix:///var/run/logd.sock')
