def is_unix_socket(hostname):
if isinstance(hostname, six.string_types) and os.path.exists(hostname):
mode = os.stat(hostname).st_mode
return stat.S_ISSOCK(mode)
return False
raised.
Returns a socket object (not a list of socket objects like
`bind_sockets`)
"""
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
set_close_exec(sock.fileno())
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setblocking(0)
try:
st = os.stat(file)
except OSError, err:
if err.errno != errno.ENOENT:
raise
else:
if stat.S_ISSOCK(st.st_mode):
os.remove(file)
else:
raise ValueError("File %s exists and is not a socket", file)
sock.bind(file)
os.chmod(file, mode)
sock.listen(backlog)
return sock
def add_accept_handler(sock, callback, io_loop=None):
"""Adds an ``IOLoop`` event handler to accept new connections on ``sock``.
When a connection is accepted, ``callback(connection, address)`` will
be run (``connection`` is a socket object, and ``address`` is the
address of the other end of the connection). Note that this signature
def unix_item_ls(the_path, ls_format, root_folder=None):
import grp
import pwd
the_parts = dict()
the_error = None
the_path_str = os.fspath(the_path)
if 'p' in ls_format:
the_parts['p'] = the_path_str
elif 'P' in ls_format:
the_parts['P'] = the_path_str
try:
the_stats = os.lstat(the_path)
for format_char in ls_format:
if format_char == 'I':
the_parts[format_char] = the_stats[stat.ST_INO] # inode number
elif format_char == 'R':
the_parts[format_char] = utils.unix_permissions_to_str(
the_stats.st_mode) # permissions
elif format_char == 'L':
the_parts[format_char] = the_stats[stat.ST_NLINK] # num links
elif format_char == 'u':
try:
the_parts[format_char] = str(the_stats[stat.ST_UID])[
0] # unknown user name, get the number
except Exception:
the_parts[format_char] = "no_uid"
elif format_char == 'U':
try:
the_parts[format_char] = pwd.getpwuid(
the_stats[stat.ST_UID])[0] # user
except KeyError:
the_parts[format_char] = str(the_stats[stat.ST_UID])[
0] # unknown user name, get the number
except Exception:
the_parts[format_char] = "no_uid"
elif format_char == 'g':
try:
the_parts[format_char] = str(the_stats[stat.ST_GID])[
0] # unknown group name, get the number
except Exception:
the_parts[format_char] = "no_gid"
elif format_char == 'G':
try:
the_parts[format_char] = grp.getgrgid(
the_stats[stat.ST_GID])[0] # group
except KeyError:
the_parts[format_char] = str(the_stats[stat.ST_GID])[
0] # unknown group name, get the number
except Exception:
the_parts[format_char] = "no_gid"
elif format_char == 'S':
the_parts[format_char] = the_stats[
stat.ST_SIZE] # size in bytes
elif format_char == 'T':
the_parts[format_char] = time.strftime(
"%Y/%m/%d-%H:%M:%S", time.gmtime(
(the_stats[stat.ST_MTIME]))) # modification time
elif format_char == 'C':
if not (stat.S_ISLNK(the_stats.st_mode)
or stat.S_ISDIR(the_stats.st_mode)):
the_parts[format_char] = utils.get_file_checksum(the_path)
else:
the_parts[format_char] = ""
elif format_char == 'P' or format_char == 'p':
path_to_return = the_path_str
if format_char == 'p' and root_folder is not None:
path_to_return = os.path.relpath(the_path,
start=root_folder)
# E will bring us Extra data (path postfix) but we want to know if it's DIR in any case
if stat.S_ISDIR(the_stats.st_mode) and 'D' in ls_format:
path_to_return += '/'
if 'E' in ls_format:
if stat.S_ISLNK(the_stats.st_mode):
path_to_return += '@'
elif not stat.S_ISDIR(the_stats.st_mode) and (
the_stats.st_mode &
(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)):
path_to_return += '*'
elif stat.S_ISSOCK(the_stats.st_mode):
path_to_return += '='
elif stat.S_ISFIFO(the_stats.st_mode):
path_to_return += '|'
the_parts[format_char] = path_to_return
elif format_char == 'a' or format_char == 'f':
import subprocess
completed_process = subprocess.run(f'ls -lO "{the_path_str}"',
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if completed_process.returncode != 0:
the_parts[format_char] = utils.unicodify(
completed_process.stderr)
else:
ls_line = utils.unicodify(completed_process.stdout)
flag_matches = re.findall(
"arch|archived|opaque|nodump|sappnd|sappend|schg|schange|simmutable|uappnd|uappend|uchg|uchange|uimmutable|hidden",
ls_line)
if flag_matches:
the_parts[format_char] = ",".join(flag_matches)
else:
the_parts[format_char] = "[]"
except Exception as ex:
the_error = [the_path_str, ex.strerror]
return the_parts, the_error
def color_file(file_path: str, path_stat: os.stat_result) -> (Color, str):
"""Determine color to use for file *approximately* as ls --color would,
given lstat() results and its path.
Parameters
----------
file_path:
relative path of file (as user typed it).
path_stat:
lstat() results for file_path.
Returns
-------
color token, color_key
Notes
-----
* implementation follows one authority:
https://github.com/coreutils/coreutils/blob/master/src/ls.c#L4879
* except:
1. does not return 'mi'. That's the color ls uses to show the (missing) *target* of a symlink
(in ls -l, not ls).
2. in dircolors, setting type code to '0 or '00' bypasses that test and proceeds to others.
In our implementation, setting code to '00' paints the file with no color.
This is arguably a bug.
"""
lsc = builtins.__xonsh__.env["LS_COLORS"]
color_key = "fi"
# if symlink, get info on (final) target
if stat.S_ISLNK(path_stat.st_mode):
try:
tar_path_stat = os.stat(file_path) # and work with its properties
if lsc.is_target("ln"): # if ln=target
path_stat = tar_path_stat
except FileNotFoundError: # bug always color broken link 'or'
color_key = "or" # early exit
ret_color_token = file_color_tokens.get(color_key, Text)
return ret_color_token, color_key
mode = path_stat.st_mode
if stat.S_ISREG(mode):
if mode & stat.S_ISUID:
color_key = "su"
elif mode & stat.S_ISGID:
color_key = "sg"
else:
cap = os_listxattr(file_path, follow_symlinks=False)
if cap and "security.capability" in cap: # protect None return on some OS?
color_key = "ca"
elif stat.S_IMODE(mode) & (stat.S_IXUSR + stat.S_IXGRP +
stat.S_IXOTH):
color_key = "ex"
elif path_stat.st_nlink > 1:
color_key = "mh"
else:
color_key = "fi"
elif stat.S_ISDIR(
mode): # ls --color doesn't colorize sticky or ow if not dirs...
color_key = "di"
if not (ON_WINDOWS
): # on Windows, these do not mean what you think they mean.
if (mode & stat.S_ISVTX) and (mode & stat.S_IWOTH):
color_key = "tw"
elif mode & stat.S_IWOTH:
color_key = "ow"
elif mode & stat.S_ISVTX:
color_key = "st"
elif stat.S_ISLNK(mode):
color_key = "ln"
elif stat.S_ISFIFO(mode):
color_key = "pi"
elif stat.S_ISSOCK(mode):
color_key = "so"
elif stat.S_ISBLK(mode):
color_key = "bd"
elif stat.S_ISCHR(mode):
color_key = "cd"
elif stat.S_ISDOOR(mode):
color_key = "do"
else:
color_key = "or" # any other type --> orphan
# if still normal file -- try color by file extension.
# note: symlink to *.<ext> will be colored 'fi' unless the symlink itself
# ends with .<ext>. `ls` does the same. Bug-for-bug compatibility!
if color_key == "fi":
match = color_file_extension_RE.match(file_path)
if match:
ext = "*" + match.group(1) # look for *.<fileExtension> coloring
if ext in lsc:
color_key = ext
ret_color_token = file_color_tokens.get(color_key, Text)
return ret_color_token, color_key
def process(path):
s = os.lstat(path)
if stat.S_ISDIR(s.st_mode):
update_hash('d')
elif stat.S_ISCHR(s.st_mode):
update_hash('c')
elif stat.S_ISBLK(s.st_mode):
update_hash('b')
elif stat.S_ISSOCK(s.st_mode):
update_hash('s')
elif stat.S_ISLNK(s.st_mode):
update_hash('l')
elif stat.S_ISFIFO(s.st_mode):
update_hash('p')
else:
update_hash('-')
def add_perm(mask, on, off='-'):
if mask & s.st_mode:
update_hash(on)
else:
update_hash(off)
add_perm(stat.S_IRUSR, 'r')
add_perm(stat.S_IWUSR, 'w')
if stat.S_ISUID & s.st_mode:
add_perm(stat.S_IXUSR, 's', 'S')
else:
add_perm(stat.S_IXUSR, 'x')
add_perm(stat.S_IRGRP, 'r')
add_perm(stat.S_IWGRP, 'w')
if stat.S_ISGID & s.st_mode:
add_perm(stat.S_IXGRP, 's', 'S')
else:
add_perm(stat.S_IXGRP, 'x')
add_perm(stat.S_IROTH, 'r')
add_perm(stat.S_IWOTH, 'w')
if stat.S_ISVTX & s.st_mode:
update_hash('t')
else:
add_perm(stat.S_IXOTH, 'x')
if include_owners:
update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name)
update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name)
update_hash(" ")
if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode):
update_hash("%9s" % ("%d.%d" % (os.major(s.st_rdev), os.minor(s.st_rdev))))
else:
update_hash(" " * 9)
update_hash(" ")
if stat.S_ISREG(s.st_mode):
update_hash("%10d" % s.st_size)
else:
update_hash(" " * 10)
update_hash(" ")
fh = hashlib.sha256()
if stat.S_ISREG(s.st_mode):
# Hash file contents
with open(path, 'rb') as d:
for chunk in iter(lambda: d.read(4096), b""):
fh.update(chunk)
update_hash(fh.hexdigest())
else:
update_hash(" " * len(fh.hexdigest()))
update_hash(" %s" % path)
if stat.S_ISLNK(s.st_mode):
update_hash(" -> %s" % os.readlink(path))
update_hash("\n")
def DoUnaryOp(op_id, s):
# type: (Id_t, str) -> bool
# Only use lstat if we're testing for a symlink.
if op_id in (Id.BoolUnary_h, Id.BoolUnary_L):
try:
mode = posix.lstat(s).st_mode
except OSError:
# TODO: simple_test_builtin should this as status=2.
#e_die("lstat() error: %s", e, word=node.child)
return False
return stat.S_ISLNK(mode)
try:
st = posix.stat(s)
except OSError as e:
# TODO: simple_test_builtin should this as status=2.
# Problem: we really need errno, because test -f / is bad argument,
# while test -f /nonexistent is a good argument but failed. Gah.
# ENOENT vs. ENAMETOOLONG.
#e_die("stat() error: %s", e, word=node.child)
return False
mode = st.st_mode
if op_id in (Id.BoolUnary_e, Id.BoolUnary_a): # -a is alias for -e
return True
if op_id == Id.BoolUnary_f:
return stat.S_ISREG(mode)
if op_id == Id.BoolUnary_d:
return stat.S_ISDIR(mode)
if op_id == Id.BoolUnary_b:
return stat.S_ISBLK(mode)
if op_id == Id.BoolUnary_c:
return stat.S_ISCHR(mode)
if op_id == Id.BoolUnary_p:
return stat.S_ISFIFO(mode)
if op_id == Id.BoolUnary_S:
return stat.S_ISSOCK(mode)
if op_id == Id.BoolUnary_x:
return posix.access(s, posix.X_OK_)
if op_id == Id.BoolUnary_r:
return posix.access(s, posix.R_OK_)
if op_id == Id.BoolUnary_w:
return posix.access(s, posix.W_OK_)
if op_id == Id.BoolUnary_s:
return st.st_size != 0
if op_id == Id.BoolUnary_O:
return st.st_uid == posix.geteuid()
if op_id == Id.BoolUnary_G:
return st.st_gid == posix.getegid()
e_die("%s isn't implemented", op_id) # implicit location
def is_socket(path):
# type: (str) -> bool
return stat.S_ISSOCK(os.lstat(path).st_mode)
def color_file(file_path: str, mode: int) -> (Color, str):
"""Determine color to use for file as ls -c would, given stat() results and its name.
Parameters
----------
file_path : string
relative path of file (as user typed it).
mode : int
stat() results for file_path.
Returns
-------
color token, color_key
Notes
-----
* doesn't handle CA (capability)
* doesn't handle LS TARGET mapping
"""
lsc = builtins.__xonsh__.env["LS_COLORS"]
color_key = "rs"
if stat.S_ISLNK(mode): # must test link before S_ISREG (esp execute)
color_key = "ln"
try:
os.stat(file_path)
except FileNotFoundError:
color_key = "or"
elif stat.S_ISREG(mode):
if stat.S_IMODE(mode) & (stat.S_IXUSR + stat.S_IXGRP + stat.S_IXOTH):
color_key = "ex"
elif (
mode & stat.S_ISUID
): # too many tests before we get to the common case -- restructure?
color_key = "su"
elif mode & stat.S_ISGID:
color_key = "sg"
else:
match = color_file_extension_RE.match(file_path)
if match:
ext = "*" + match.group(
1) # look for *.<fileExtension> coloring
if ext in lsc:
color_key = ext
else:
color_key = "rs"
else:
color_key = "rs"
elif stat.S_ISDIR(
mode): # ls -c doesn't colorize sticky or ow if not dirs...
color_key = ("di", "ow", "st",
"tw")[(mode & stat.S_ISVTX == stat.S_ISVTX) * 2 +
(mode & stat.S_IWOTH == stat.S_IWOTH)]
elif stat.S_ISCHR(mode):
color_key = "cd"
elif stat.S_ISBLK(mode):
color_key = "bd"
elif stat.S_ISFIFO(mode):
color_key = "pi"
elif stat.S_ISSOCK(mode):
color_key = "so"
elif stat.S_ISDOOR(mode):
color_key = "do" # bug missing mapping for FMT based PORT and WHITEOUT ??
ret_color_token = file_color_tokens.get(color_key, Text)
return ret_color_token, color_key
def _remove_existed_sock(self, sock_file):
if os.path.exists(sock_file):
mode = os.stat(sock_file).st_mode
if stat.S_ISSOCK(mode):
os.remove(sock_file)
def format_output(module,
path,
st,
follow,
get_md5,
get_checksum,
checksum_algorithm,
mimetype=None,
charset=None):
mode = st.st_mode
# back to ansible
output = dict(
exists=True,
path=path,
mode="%04o" % stat.S_IMODE(mode),
isdir=stat.S_ISDIR(mode),
ischr=stat.S_ISCHR(mode),
isblk=stat.S_ISBLK(mode),
isreg=stat.S_ISREG(mode),
isfifo=stat.S_ISFIFO(mode),
islnk=stat.S_ISLNK(mode),
issock=stat.S_ISSOCK(mode),
uid=st.st_uid,
gid=st.st_gid,
size=st.st_size,
inode=st.st_ino,
dev=st.st_dev,
nlink=st.st_nlink,
atime=st.st_atime,
mtime=st.st_mtime,
ctime=st.st_ctime,
wusr=bool(mode & stat.S_IWUSR),
rusr=bool(mode & stat.S_IRUSR),
xusr=bool(mode & stat.S_IXUSR),
wgrp=bool(mode & stat.S_IWGRP),
rgrp=bool(mode & stat.S_IRGRP),
xgrp=bool(mode & stat.S_IXGRP),
woth=bool(mode & stat.S_IWOTH),
roth=bool(mode & stat.S_IROTH),
xoth=bool(mode & stat.S_IXOTH),
isuid=bool(mode & stat.S_ISUID),
isgid=bool(mode & stat.S_ISGID),
readable=os.access(path, os.R_OK),
writeable=os.access(path, os.W_OK),
excutable=os.access(path, os.X_OK),
)
if stat.S_ISLNK(mode):
output['lnk_source'] = os.path.realpath(path)
if stat.S_ISREG(mode) and get_md5 and os.access(path, os.R_OK):
# Will fail on FIPS-140 compliant systems
try:
output['md5'] = module.md5(path)
except ValueError:
output['md5'] = None
if stat.S_ISREG(mode) and get_checksum and os.access(path, os.R_OK):
output['checksum'] = module.digest_from_file(path, checksum_algorithm)
try:
pw = pwd.getpwuid(st.st_uid)
output['pw_name'] = pw.pw_name
grp_info = grp.getgrgid(st.st_gid)
output['gr_name'] = grp_info.gr_name
except:
pass
if not (mimetype is None and charset is None):
output['mime_type'] = mimetype
output['charset'] = charset
return output
def is_socket(self):
return stat.S_ISSOCK(self.filetype) != 0
def _is_special_file(fstat):
"""
Returns true if the file is a special one (i.e. stdout, stderr, null....)
"""
return stat.S_ISFIFO(fstat.st_mode) or stat.S_ISCHR(fstat.st_mode) or stat.S_ISSOCK(fstat.st_mode)
def handle_test():
#TODO: this doesn't have use a socket like the above. We could rewrite it to spawn suricata and read its output instead (see handle_validate())
#TODO update: I tried this, but it's a bit difficult because of how much data suricata wants to pump through stdout.
# It seems communicate() results in some messages being dropped due to volume or the socket being closed
#We have to spawn a new suricata instance in order to change the rule file that is being used
# Which also means we have to set up a new logging directory and everything
tmp_work_dir = tempfile.TemporaryDirectory(dir="/dev/shm/")
os.mkdir(os.path.join(tmp_work_dir.name, "logs"))
tmp_cmd_sock_path = os.path.join(tmp_work_dir.name, "suricata.sock")
tmp_output_sock_path = os.path.join(tmp_work_dir.name, "logs", "eve.sock")
rule_file_path = os.path.join(tmp_work_dir.name, "suricata.rules")
with open(rule_file_path, 'w') as f:
f.write(request.form.get("rules"))
for lua in request.files.getlist("lua[]"):
with open(
os.path.join(tmp_work_dir.name,
os.path.basename(lua.filename)), 'w') as lua_file:
logging.debug("Writing to {}".format(
os.path.join(tmp_work_dir.name,
os.path.basename(lua.filename))))
lua_file.write(lua.read().decode("utf-8"))
#suricata_process = subprocess.Popen(['suricata', '-c', './config/suricata.yaml', '--set', 'default-rule-path={}'.format(tmp_work_dir.name), '--unix-socket={}'.format(tmp_cmd_sock_path)])
suricata_process = subprocess.Popen([
'suricata', '-c', './config/suricata.yaml', '--set',
'default-rule-path={}'.format(
tmp_work_dir.name), '--unix-socket={}'.format(tmp_cmd_sock_path)
],
stdout=subprocess.DEVNULL)
#Create the socket that we'll read from and connect to it
tmp_output_sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
tmp_output_sock.bind(tmp_output_sock_path)
tmp_output_sock.settimeout(10)
#Wait for the suricata process to start up
tmp_cmd_sock = None
while tmp_cmd_sock is None:
try:
if stat.S_ISSOCK(os.stat(tmp_cmd_sock_path).st_mode):
tmp_cmd_sock = socket.socket(socket.AF_UNIX)
tmp_cmd_sock.connect(tmp_cmd_sock_path)
tmp_cmd_sock.settimeout(10)
#Apparently we have to send a version when we connect or suricata won't accept commands
send(tmp_cmd_sock, {"version": '0.2'})
else:
raise FileNotFoundError()
except FileNotFoundError:
logging.debug("Waiting for {} to exist".format(tmp_cmd_sock_path))
time.sleep(0.1)
alerts_and_metadata = process_pcap(request.files.get("pcap").stream,
get_files=False,
work_dir=tmp_work_dir.name,
command_sock=tmp_cmd_sock,
output_sock=tmp_output_sock)
send_command(tmp_cmd_sock, "shutdown")
tmp_work_dir.cleanup()
alerted = {}
for record in alerts_and_metadata:
if 'event_type' in record and record[
'event_type'] == 'alert' and 'alert' in record:
if record['alert']['signature'] not in alerted:
alerted[record['alert']['signature']] = 0
alerted[record['alert']['signature']] += 1
return json.dumps(alerted)
suricata_process = subprocess.Popen([
'suricata', '-c', './config/suricata.yaml',
'--unix-socket={}'.format(command_sock_path)
],
stdout=subprocess.DEVNULL)
#Create the socket that we'll read from and connect to it
output_sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
output_sock.bind(output_sock_path)
output_sock.settimeout(10)
#Wait for the suricata process to start up
command_sock = None
while command_sock is None:
try:
if stat.S_ISSOCK(os.stat(command_sock_path).st_mode):
command_sock = socket.socket(socket.AF_UNIX)
command_sock.connect(command_sock_path)
command_sock.settimeout(10)
#Apparently we have to send a version when we connect or suricata won't accept commands
send(command_sock, {"version": '0.2'})
else:
raise FileNotFoundError()
except FileNotFoundError:
logging.info("Waiting for {} to exist".format(command_sock_path))
time.sleep(1.0)
def process_pcap(pcap_file,
get_files=False,
def __init__(self,
host="localhost",
user=None,
passwd="",
db=None,
port=3306,
unix_socket=None,
charset='',
sql_mode=None,
read_default_file=None,
use_unicode=None,
client_flag=0,
cursorclass=Cursor,
init_command=None,
connect_timeout=None,
ssl=None,
read_default_group=None,
compress=None,
named_pipe=None,
conv=decoders,
encoders=encoders):
"""
Establish a connection to the MySQL database. Accepts several
arguments:
host: Host where the database server is located
user: Username to log in as
passwd: Password to use.
db: Database to use, None to not use a particular one.
port: MySQL port to use, default is usually OK.
unix_socket: Optionally, you can use a unix socket rather than TCP/IP.
charset: Charset you want to use.
sql_mode: Default SQL_MODE to use.
read_default_file: Specifies my.cnf file to read these parameters from under the [client] section.
conv: Decoders dictionary to use instead of the default one. This is used to provide custom marshalling of types. See converters.
use_unicode: Whether or not to default to unicode strings. This option defaults to true for Py3k.
client_flag: Custom flags to send to MySQL. Find potential values in constants.CLIENT.
cursorclass: Custom cursor class to use.
init_command: Initial SQL statement to run when connection is established.
connect_timeout: Timeout before throwing an exception when connecting.
ssl: A dict of arguments similar to mysql_ssl_set()'s parameters. For now the capath and cipher arguments are not supported.
read_default_group: Group to read from in the configuration file.
compress; Not supported
named_pipe: Not supported
"""
if use_unicode is None and sys.version_info[0] > 2:
use_unicode = True
if compress or named_pipe:
raise NotImplementedError(
"compress and named_pipe arguments are not supported")
if ssl and ('capath' in ssl or 'cipher' in ssl):
raise NotImplementedError(
'ssl options capath and cipher are not supported')
self.socket = None
self.ssl = False
if ssl:
self.ssl = True
client_flag |= SSL
for k in ('key', 'cert', 'ca'):
v = None
if k in ssl:
v = ssl[k]
setattr(self, k, v)
if read_default_group and not read_default_file:
if sys.platform.startswith("win"):
read_default_file = "c:\\my.ini"
else:
for f in ('~/.my.cnf', '/etc/my.cnf', '/etc/mysql/my.cnf'):
if os.path.isfile(os.path.expanduser(f)):
read_default_file = f
break
if read_default_file:
if not read_default_group:
read_default_group = "client"
cfg = RawConfigParser()
cfg.read(os.path.expanduser(read_default_file))
def _config(key, default):
try:
return cfg.get(read_default_group, key)
except:
return default
user = _config("user", user)
passwd = _config("password", passwd)
host = _config("host", host)
db = _config("db", db)
unix_socket = _config("socket", unix_socket)
port = _config("port", port)
charset = _config("default-character-set", charset)
if (host == 'localhost' and port == 3306
and not sys.platform.startswith('win')
and (unix_socket is None or not os.path.exists(unix_socket))):
for f in ('/var/lib/mysql/mysql.sock', '/var/run/mysql/mysql.sock',
'/var/run/mysql.sock', '/var/mysql/mysql.sock'):
if os.path.exists(f) and stat.S_ISSOCK(os.stat(f).st_mode):
unix_socket = f
break
self.host = host
self.port = port
self.user = user or DEFAULT_USER
if isinstance(passwd, bytes):
passwd = passwd.decode(charset if charset else DEFAULT_CHARSET)
self.password = passwd
self.db = db
self.unix_socket = unix_socket
self.conv = conv
self.encoders = encoders
if charset:
self.charset = charset
self.use_unicode = True
else:
self.charset = DEFAULT_CHARSET
self.use_unicode = False
if use_unicode is not None:
self.use_unicode = use_unicode
client_flag |= CAPABILITIES
client_flag |= MULTI_STATEMENTS
if self.db:
client_flag |= CONNECT_WITH_DB
# self.client_flag |= CLIENT_DEPRECATE_EOF
self.client_flag = client_flag
self.cursorclass = cursorclass
self.connect_timeout = connect_timeout
self._connect()
self.messages = []
self.set_charset(charset)
self._result = None
self.host_info = "Not connected"
self.autocommit(False)
if sql_mode is not None:
c = self.cursor()
c.execute("SET sql_mode=%s", (sql_mode, ))
self.commit()
if init_command is not None:
c = self.cursor()
c.execute(init_command)
self.commit()
def copy_all(self):
"""Core copy process. This is the most important step of this
stage. It clones live filesystem into a local partition in the
selected hard disk."""
self.db.progress('START', 0, 100, 'ubiquity/install/title')
self.db.progress('INFO', 'ubiquity/install/copying')
fs_size = os.path.join(self.casper_path, 'filesystem.size')
if os.path.exists(fs_size):
with open(fs_size) as total_size_fp:
total_size = int(total_size_fp.readline())
else:
# Fallback in case an Linux Mint derivative forgets to put
# /casper/filesystem.size on the CD, or to account for things
# like CD->USB transformation tools that don't copy this file.
# This is slower than just reading the size from a file, but
# better than crashing.
#
# Obviously doing os.walk() twice is inefficient, but I'd rather
# not suck the list into ubiquity's memory, and I'm guessing
# that the kernel's dentry cache will avoid most of the slowness
# anyway.
total_size = 0
for dirpath, dirnames, filenames in os.walk(self.source):
for name in dirnames + filenames:
fqpath = os.path.join(dirpath, name)
total_size += os.lstat(fqpath).st_size
# Progress bar handling:
# We sample progress every half-second (assuming time.time() gives
# us sufficiently good granularity) and use the average of progress
# over the last minute or so to decide how much time remains. We
# don't bother displaying any progress for the first ten seconds in
# order to allow things to settle down, and we only update the "time
# remaining" indicator at most every two seconds after that.
copy_progress = 0
copied_size = 0
directory_times = []
time_start = time.time()
times = [(time_start, copied_size)]
long_enough = False
time_last_update = time_start
debug = 'UBIQUITY_DEBUG' in os.environ
if self.db.get('ubiquity/install/md5_check') == 'false':
md5_check = False
else:
md5_check = True
# Increase kernel flush times during bulk data copying to make it
# more likely that small files are packed contiguously, which should
# speed up initial boot times.
dirty_writeback_centisecs = None
dirty_expire_centisecs = None
if os.path.exists('/proc/sys/vm/dirty_writeback_centisecs'):
with open('/proc/sys/vm/dirty_writeback_centisecs') as dwc:
dirty_writeback_centisecs = int(dwc.readline())
with open('/proc/sys/vm/dirty_writeback_centisecs', 'w') as dwc:
print('3000\n', file=dwc)
if os.path.exists('/proc/sys/vm/dirty_expire_centisecs'):
with open('/proc/sys/vm/dirty_expire_centisecs') as dec:
dirty_expire_centisecs = int(dec.readline())
with open('/proc/sys/vm/dirty_expire_centisecs', 'w') as dec:
print('6000\n', file=dec)
old_umask = os.umask(0)
for dirpath, dirnames, filenames in os.walk(self.source):
sp = dirpath[len(self.source) + 1:]
for name in dirnames + filenames:
relpath = os.path.join(sp, name)
# /etc/fstab was legitimately created by partman, and
# shouldn't be copied again. Similarly, /etc/crypttab may
# have been legitimately created by the user-setup plugin.
if relpath in ("etc/fstab", "etc/crypttab"):
continue
sourcepath = os.path.join(self.source, relpath)
targetpath = os.path.join(self.target, relpath)
st = os.lstat(sourcepath)
# Is the path blacklisted?
if (not stat.S_ISDIR(st.st_mode) and
'/%s' % relpath in self.blacklist):
if debug:
syslog.syslog('Not copying %s' % relpath)
continue
# Remove the target if necessary and if we can.
install_misc.remove_target(
self.source, self.target, relpath, st)
# Now actually copy source to target.
mode = stat.S_IMODE(st.st_mode)
if stat.S_ISLNK(st.st_mode):
linkto = os.readlink(sourcepath)
os.symlink(linkto, targetpath)
elif stat.S_ISDIR(st.st_mode):
if not os.path.isdir(targetpath):
try:
os.mkdir(targetpath, mode)
except OSError as e:
# there is a small window where update-apt-cache
# can race with us since it creates
# "/target/var/cache/apt/...". Hence, ignore
# failure if the directory does now exist where
# brief moments before it didn't.
if e.errno != errno.EEXIST:
raise
elif stat.S_ISCHR(st.st_mode):
os.mknod(targetpath, stat.S_IFCHR | mode, st.st_rdev)
elif stat.S_ISBLK(st.st_mode):
os.mknod(targetpath, stat.S_IFBLK | mode, st.st_rdev)
elif stat.S_ISFIFO(st.st_mode):
os.mknod(targetpath, stat.S_IFIFO | mode)
elif stat.S_ISSOCK(st.st_mode):
os.mknod(targetpath, stat.S_IFSOCK | mode)
elif stat.S_ISREG(st.st_mode):
install_misc.copy_file(
self.db, sourcepath, targetpath, md5_check)
# Copy metadata.
copied_size += st.st_size
os.lchown(targetpath, st.st_uid, st.st_gid)
if not stat.S_ISLNK(st.st_mode):
os.chmod(targetpath, mode)
if stat.S_ISDIR(st.st_mode):
directory_times.append(
(targetpath, st.st_atime, st.st_mtime))
# os.utime() sets timestamp of target, not link
elif not stat.S_ISLNK(st.st_mode):
try:
os.utime(targetpath, (st.st_atime, st.st_mtime))
except Exception:
# We can live with timestamps being wrong.
pass
if (hasattr(os, "listxattr") and
hasattr(os, "supports_follow_symlinks") and
os.supports_follow_symlinks):
try:
attrnames = os.listxattr(
sourcepath, follow_symlinks=False)
for attrname in attrnames:
attrvalue = os.getxattr(
sourcepath, attrname, follow_symlinks=False)
os.setxattr(
targetpath, attrname, attrvalue,
follow_symlinks=False)
except OSError as e:
if e.errno not in (
errno.EPERM, errno.ENOTSUP, errno.ENODATA):
raise
if int((copied_size * 90) / total_size) != copy_progress:
copy_progress = int((copied_size * 90) / total_size)
self.db.progress('SET', 10 + copy_progress)
time_now = time.time()
if (time_now - times[-1][0]) >= 0.5:
times.append((time_now, copied_size))
if not long_enough and time_now - times[0][0] >= 10:
long_enough = True
if long_enough and time_now - time_last_update >= 2:
time_last_update = time_now
while (time_now - times[0][0] > 60 and
time_now - times[1][0] >= 60):
times.pop(0)
speed = ((times[-1][1] - times[0][1]) /
(times[-1][0] - times[0][0]))
if speed != 0:
time_remaining = (
int((total_size - copied_size) / speed))
if time_remaining < 60:
self.db.progress(
'INFO', 'ubiquity/install/copying_minute')
# Apply timestamps to all directories now that the items within them
# have been copied.
for dirtime in directory_times:
(directory, atime, mtime) = dirtime
try:
os.utime(directory, (atime, mtime))
except Exception:
# I have no idea why I've been getting lots of bug reports
# about this failing, but I really don't care. Ignore it.
pass
# Revert to previous kernel flush times.
if dirty_writeback_centisecs is not None:
with open('/proc/sys/vm/dirty_writeback_centisecs', 'w') as dwc:
print(dirty_writeback_centisecs, file=dwc)
if dirty_expire_centisecs is not None:
with open('/proc/sys/vm/dirty_expire_centisecs', 'w') as dec:
print(dirty_expire_centisecs, file=dec)
# Try some possible locations for the kernel we used to boot. This
# lets us save a couple of megabytes of CD space.
bootdir = self.target_file('boot')
kernel = self.find_cd_kernel()
if kernel:
prefix = os.path.basename(kernel).split('-', 1)[0]
release = os.uname()[2]
target_kernel = os.path.join(bootdir, '%s-%s' % (prefix, release))
copies = []
# ISO9660 images may have to use .efi rather than .efi.signed in
# order to support being booted using isolinux, which must abide
# by archaic 8.3 restrictions.
for suffix in (".efi", ".efi.signed"):
if os.path.exists(kernel + suffix):
signed_kernel = kernel + suffix
break
else:
signed_kernel = None
if os.path.exists(kernel):
copies.append((kernel, target_kernel))
elif signed_kernel is not None:
# No unsigned kernel. We'll construct it using sbsigntool.
copies.append((signed_kernel, target_kernel))
if signed_kernel is not None:
copies.append((signed_kernel, "%s.efi.signed" % target_kernel))
for source, target in copies:
osextras.unlink_force(target)
install_misc.copy_file(self.db, source, target, md5_check)
os.lchown(target, 0, 0)
os.chmod(target, 0o644)
st = os.lstat(source)
try:
os.utime(target, (st.st_atime, st.st_mtime))
except Exception:
# We can live with timestamps being wrong.
pass
if not os.path.exists(kernel) and signed_kernel is not None:
# Construct the unsigned kernel.
subprocess.check_call(["sbattach", "--remove", target_kernel])
os.umask(old_umask)
self.db.progress('SET', 100)
self.db.progress('STOP')
def __is_socket(cls, path):
return stat.S_ISSOCK(
os.stat(path).st_mode) if os.path.exists(path) else False
def register_unix():
"""Register the plugins that use a unix socket for communication.
Unix plugins can be configured in a variety of ways:
1.) Listed in the configuration file under plugin.unix
2.) Via environment variable
2.) Automatically, by placing the socket in the default socket directory
Here, we will parse the configurations and the default socket directory,
add them to the PluginManager, and return a unified list of all known
unix-configured plugins.
Returns:
list[str]: The ids of all plugins that were registered.
"""
registered = []
configured = config.options.get('plugin.unix', [])
if not configured:
logger.info(_('No plugin configurations for unix'))
logger.debug(_('unix plugin configuration: {}').format(configured))
for address in configured:
# The config here should be the path the the unix socket, which is our address.
# First, check that the socket exists and that the address is a socket file.
if not os.path.exists(address):
logger.error(_('Socket {} not found').format(address))
continue
if not stat.S_ISSOCK(os.stat(address).st_mode):
logger.error(_('{} is not a socket').format(address))
continue
plugin_id = register_plugin(address, 'unix')
if plugin_id is None:
logger.error(_('Failed to register plugin with address: {}').format(address))
continue
registered.append(plugin_id)
# Now, go through the default socket directory and pick up any sockets that
# may be set for automatic registration.
if not os.path.exists(const.SOCKET_DIR):
logger.debug(
_('No default socket path found, no plugins will be registered from {}')
.format(const.SOCKET_DIR)
)
else:
logger.debug(
_('Registering plugins from default socket directory ({})')
.format(const.SOCKET_DIR)
)
for item in os.listdir(const.SOCKET_DIR):
logger.debug(' {}'.format(item))
address = os.path.join(const.SOCKET_DIR, item)
# Check if the file is a socket
if not stat.S_ISSOCK(os.stat(address).st_mode):
logger.debug(_('{} is not a socket - skipping').format(address))
continue
plugin_id = register_plugin(address, 'unix')
if plugin_id is None:
logger.error(_('Failed to register plugin with address: {}').format(address))
continue
registered.append(plugin_id)
# We want the plugins registered from this default directory to
# be surfaced in the config, so we will add it there.
if config.options.get('plugin.unix') is None:
config.options.set('plugin.unix', [address])
else:
config.options.get('plugin.unix').append(address)
logger.info('Registered unix plugins: {}'.format(registered))
return registered
# Author: Niels de Vos <*****@*****.**>
#
from __future__ import print_function
import os
import stat
import sys
import socket
ret = 1
if len(sys.argv) != 2:
print('Usage: %s <socket>' % (sys.argv[0]))
sys.exit(ret)
path = sys.argv[1]
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind(path)
stbuf = os.stat(path)
mode = stbuf.st_mode
if stat.S_ISSOCK(mode):
ret = 0
sock.close()
os.unlink(path)
sys.exit(ret)
def _is_socket(self, item):
mode = os.stat(os.path.join(self.datadir, item)).st_mode
if stat.S_ISSOCK(mode):
return True
return False
def valid(self, s, partial=False):
mode = os.stat(s).st_mode
if not stat.S_ISSOCK(mode):
raise ArgumentValid('socket path {0} is not a socket'.format(s))
self.val = s
def FileFromFilesystem(path, pathId, possibleMatch = None, inodeInfo = False,
assumeRoot=False, statBuf=None, sha1FailOk=False):
if statBuf:
s = statBuf
else:
s = os.lstat(path)
global userCache, groupCache, _havePrelink
if assumeRoot:
owner = 'root'
group = 'root'
elif isinstance(s.st_uid, basestring):
# Already stringified -- some capsule code will fabricate a stat result
# from e.g. a RPM header
owner = s.st_uid
group = s.st_gid
else:
# + is not a valid char in user/group names; if the uid is not mapped
# to a user, prepend it with + and store it as a string
try:
owner = userCache.lookupId('/', s.st_uid)
except KeyError:
owner = '+%d' % s.st_uid
try:
group = groupCache.lookupId('/', s.st_gid)
except KeyError:
group = '+%d' % s.st_gid
needsSha1 = 0
inode = InodeStream(s.st_mode & 07777, s.st_mtime, owner, group)
if (stat.S_ISREG(s.st_mode)):
f = RegularFile(pathId)
needsSha1 = 1
elif (stat.S_ISLNK(s.st_mode)):
f = SymbolicLink(pathId)
if hasattr(s, 'linkto'):
f.target.set(s.linkto)
else:
f.target.set(os.readlink(path))
elif (stat.S_ISDIR(s.st_mode)):
f = Directory(pathId)
elif (stat.S_ISSOCK(s.st_mode)):
f = Socket(pathId)
elif (stat.S_ISFIFO(s.st_mode)):
f = NamedPipe(pathId)
elif (stat.S_ISBLK(s.st_mode)):
f = BlockDevice(pathId)
f.devt.major.set(s.st_rdev >> 8)
f.devt.minor.set(s.st_rdev & 0xff)
elif (stat.S_ISCHR(s.st_mode)):
f = CharacterDevice(pathId)
f.devt.major.set(s.st_rdev >> 8)
f.devt.minor.set(s.st_rdev & 0xff)
else:
raise FilesError("unsupported file type for %s" % path)
f.inode = inode
f.flags = FlagsStream(0)
# assume we have a match if the FileMode and object type match
if possibleMatch and (possibleMatch.__class__ == f.__class__) \
and f.inode == possibleMatch.inode \
and f.inode.mtime() == possibleMatch.inode.mtime() \
and (not s.st_size or
(possibleMatch.hasContents and
s.st_size == possibleMatch.contents.size())):
f.flags.set(possibleMatch.flags())
return possibleMatch
elif (possibleMatch and (isinstance(f, RegularFile) and
isinstance(possibleMatch, RegularFile))
and (f.inode.isExecutable())
and f.inode.mtime() == possibleMatch.inode.mtime()
and f.inode.owner == possibleMatch.inode.owner
and f.inode.group == possibleMatch.inode.group
and f.inode.perms == possibleMatch.inode.perms):
# executable RegularFiles match even if there sizes are different
# as long as everything else is the same; this is to stop size
# changes from prelink from changing fileids
return possibleMatch
if needsSha1:
f.contents = RegularFileStream()
undoPrelink = False
if _havePrelink != False and f.inode.isExecutable():
try:
from conary.lib import elf
if elf.prelinked(path):
undoPrelink = True
except:
pass
if undoPrelink and _havePrelink is None:
_havePrelink = bool(os.access(PRELINK_CMD[0], os.X_OK))
if undoPrelink and _havePrelink:
prelink = subprocess.Popen(
PRELINK_CMD + ("-y", path),
stdout = subprocess.PIPE,
close_fds = True,
shell = False)
d = digestlib.sha1()
content = prelink.stdout.read()
size = 0
while content:
d.update(content)
size += len(content)
content = prelink.stdout.read()
prelink.wait()
f.contents.size.set(size)
sha1 = d.digest()
else:
try:
sha1 = sha1helper.sha1FileBin(path)
except OSError:
if sha1FailOk:
sha1 = sha1helper.sha1Empty
else:
raise
f.contents.size.set(s.st_size)
f.contents.sha1.set(sha1)
if inodeInfo:
return (f, s.st_nlink, (s.st_rdev, s.st_ino))
return f
def lsLine(name, s):
"""
Build an 'ls' line for a file ('file' in its generic sense, it
can be of any type).
"""
mode = s.st_mode
perms = array.array('B', b'-' * 10)
ft = stat.S_IFMT(mode)
if stat.S_ISDIR(ft): perms[0] = ord('d')
elif stat.S_ISCHR(ft): perms[0] = ord('c')
elif stat.S_ISBLK(ft): perms[0] = ord('b')
elif stat.S_ISREG(ft): perms[0] = ord('-')
elif stat.S_ISFIFO(ft): perms[0] = ord('f')
elif stat.S_ISLNK(ft): perms[0] = ord('l')
elif stat.S_ISSOCK(ft): perms[0] = ord('s')
else: perms[0] = ord('!')
# User
if mode & stat.S_IRUSR: perms[1] = ord('r')
if mode & stat.S_IWUSR: perms[2] = ord('w')
if mode & stat.S_IXUSR: perms[3] = ord('x')
# Group
if mode & stat.S_IRGRP: perms[4] = ord('r')
if mode & stat.S_IWGRP: perms[5] = ord('w')
if mode & stat.S_IXGRP: perms[6] = ord('x')
# Other
if mode & stat.S_IROTH: perms[7] = ord('r')
if mode & stat.S_IWOTH: perms[8] = ord('w')
if mode & stat.S_IXOTH: perms[9] = ord('x')
# Suid/sgid
if mode & stat.S_ISUID:
if perms[3] == ord('x'): perms[3] = ord('s')
else: perms[3] = ord('S')
if mode & stat.S_ISGID:
if perms[6] == ord('x'): perms[6] = ord('s')
else: perms[6] = ord('S')
if _PY3:
if isinstance(name, bytes):
name = name.decode("utf-8")
lsPerms = perms.tobytes()
lsPerms = lsPerms.decode("utf-8")
else:
lsPerms = perms.tostring()
lsresult = [
lsPerms,
str(s.st_nlink).rjust(5),
' ',
str(s.st_uid).ljust(9),
str(s.st_gid).ljust(9),
str(s.st_size).rjust(8),
' ',
]
# Need to specify the month manually, as strftime depends on locale
ttup = localtime(s.st_mtime)
sixmonths = 60 * 60 * 24 * 7 * 26
if s.st_mtime + sixmonths < time(): # Last edited more than 6mo ago
strtime = strftime("%%s %d %Y ", ttup)
else:
strtime = strftime("%%s %d %H:%M ", ttup)
lsresult.append(strtime % (_MONTH_NAMES[ttup[1]], ))
lsresult.append(name)
return ''.join(lsresult)
def setLogTarget(self, target):
try:
self.__loggingLock.acquire()
# set a format which is simpler for console use
formatter = logging.Formatter("%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s")
if target == "SYSLOG":
# Syslog daemons already add date to the message.
formatter = logging.Formatter("%(name)s[%(process)d]: %(levelname)s %(message)s")
facility = logging.handlers.SysLogHandler.LOG_DAEMON
if self.__syslogSocket == "auto":
import platform
self.__syslogSocket = self.__autoSyslogSocketPaths.get(
platform.system())
if self.__syslogSocket is not None\
and os.path.exists(self.__syslogSocket)\
and stat.S_ISSOCK(os.stat(
self.__syslogSocket).st_mode):
hdlr = logging.handlers.SysLogHandler(
self.__syslogSocket, facility=facility)
else:
logSys.error(
"Syslog socket file: %s does not exists"
" or is not a socket" % self.__syslogSocket)
return False
elif target == "STDOUT":
hdlr = logging.StreamHandler(sys.stdout)
elif target == "STDERR":
hdlr = logging.StreamHandler(sys.stderr)
else:
# Target should be a file
try:
open(target, "a").close()
hdlr = logging.handlers.RotatingFileHandler(target)
except IOError:
logSys.error("Unable to log to " + target)
logSys.info("Logging to previous target " + self.__logTarget)
return False
# Removes previous handlers -- in reverse order since removeHandler
# alter the list in-place and that can confuses the iterable
logger = getLogger("fail2ban")
for handler in logger.handlers[::-1]:
# Remove the handler.
logger.removeHandler(handler)
# And try to close -- it might be closed already
try:
handler.flush()
handler.close()
except (ValueError, KeyError): # pragma: no cover
# Is known to be thrown after logging was shutdown once
# with older Pythons -- seems to be safe to ignore there
# At least it was still failing on 2.6.2-0ubuntu1 (jaunty)
if (2,6,3) <= sys.version_info < (3,) or \
(3,2) <= sys.version_info:
raise
# tell the handler to use this format
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
# Does not display this message at startup.
if not self.__logTarget is None:
logSys.info(
"Changed logging target to %s for Fail2ban v%s"
% ((target
if target != "SYSLOG"
else "%s (%s)"
% (target, self.__syslogSocket)),
version.version))
# Sets the logging target.
self.__logTarget = target
return True
finally:
self.__loggingLock.release()
def run(request):
unit = unit_run()
option.skip_alerts = [
r'read signalfd\(4\) failed',
r'sendmsg.+failed',
r'recvmsg.+failed',
]
option.skip_sanitizer = False
_fds_info['main']['skip'] = False
_fds_info['router']['skip'] = False
_fds_info['controller']['skip'] = False
yield
# stop unit
error_stop_unit = unit_stop()
error_stop_processes = stop_processes()
# prepare log
with Log.open(encoding='utf-8') as f:
log = f.read()
Log.set_pos(f.tell())
if not option.save_log and option.restart:
shutil.rmtree(unit['temp_dir'])
Log.set_pos(0)
# clean temp_dir before the next test
if not option.restart:
_clear_conf(unit['temp_dir'] + '/control.unit.sock', log=log)
for item in os.listdir(unit['temp_dir']):
if item not in [
'control.unit.sock',
'state',
'unit.pid',
'unit.log',
]:
path = os.path.join(unit['temp_dir'], item)
public_dir(path)
if os.path.isfile(path) or stat.S_ISSOCK(
os.stat(path).st_mode):
os.remove(path)
else:
for attempt in range(10):
try:
shutil.rmtree(path)
break
except OSError as err:
if err.errno != 16:
raise
time.sleep(1)
# check descriptors
_check_fds(log=log)
# print unit.log in case of error
if hasattr(request.node, 'rep_call') and request.node.rep_call.failed:
_print_log(log)
if error_stop_unit or error_stop_processes:
_print_log(log)
# check unit.log for errors
assert error_stop_unit is None, 'stop unit'
assert error_stop_processes is None, 'stop processes'
_check_alerts(log=log)
async def create_unix_server(self,
protocol_factory,
path=None,
*,
sock=None,
backlog=100,
ssl=None,
ssl_handshake_timeout=None,
start_serving=True):
if isinstance(ssl, bool):
raise TypeError('ssl argument must be an SSLContext or None')
if ssl_handshake_timeout is not None and not ssl:
raise ValueError(
'ssl_handshake_timeout is only meaningful with ssl')
if path is not None:
if sock is not None:
raise ValueError(
'path and sock can not be specified at the same time')
path = os.fspath(path)
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# Check for abstract socket. `str` and `bytes` paths are supported.
if path[0] not in (0, '\x00'):
try:
if stat.S_ISSOCK(os.stat(path).st_mode):
os.remove(path)
except FileNotFoundError:
pass
except OSError as err:
# Directory may have permissions only to create socket.
logger.error(
'Unable to check or remove stale UNIX socket '
'%r: %r', path, err)
try:
sock.bind(path)
except OSError as exc:
sock.close()
if exc.errno == errno.EADDRINUSE:
# Let's improve the error message by adding
# with what exact address it occurs.
msg = f'Address {path!r} is already in use'
raise OSError(errno.EADDRINUSE, msg) from None
else:
raise
except:
sock.close()
raise
else:
if sock is None:
raise ValueError(
'path was not specified, and no sock specified')
if (sock.family != socket.AF_UNIX
or sock.type != socket.SOCK_STREAM):
raise ValueError(
f'A UNIX Domain Stream Socket was expected, got {sock!r}')
sock.setblocking(False)
server = base_events.Server(self, [sock], protocol_factory, ssl,
backlog, ssl_handshake_timeout)
if start_serving:
server._start_serving()
# Skip one loop iteration so that all 'loop.add_reader'
# go through.
await tasks.sleep(0, loop=self)
return server
def init_install(self, setup):
# mount the media location.
print " --> Installation started"
if (not os.path.exists("/target")):
if (setup.skip_mount):
self.error_message(message=_(
"ERROR: You must first manually mount your target filesystem(s) at /target to do a custom install!"
))
return
os.mkdir("/target")
if (not os.path.exists("/source")):
os.mkdir("/source")
# find the squashfs..
if (not os.path.exists(self.media)):
print "Base filesystem does not exist! Critical error (exiting)."
self.error_message(message=_(
"ERROR: Something is wrong with the installation medium! This is usually caused by burning tools which are not compatible with LMDE (YUMI or other multiboot tools). Please burn the ISO image to DVD/USB using a different tool."
))
return
os.system("umount --force /target/dev/shm")
os.system("umount --force /target/dev/pts")
os.system("umount --force /target/dev/")
os.system("umount --force /target/sys/")
os.system("umount --force /target/proc/")
if (not setup.skip_mount):
self.step_format_partitions(setup)
self.step_mount_partitions(setup)
else:
self.step_mount_source(setup)
# walk root filesystem
SOURCE = "/source/"
DEST = "/target/"
directory_times = []
our_total = 0
our_current = -1
os.chdir(SOURCE)
# index the files
print " --> Indexing files"
self.update_progress(pulse=True,
message=_("Indexing files to be copied.."))
for top, dirs, files in os.walk(SOURCE, topdown=False):
our_total += len(dirs) + len(files)
our_total += 1 # safenessness
print " --> Copying files"
for top, dirs, files in os.walk(SOURCE):
# Sanity check. Python is a bit schitzo
dirpath = top
if (dirpath.startswith(SOURCE)):
dirpath = dirpath[len(SOURCE):]
for name in dirs + files:
# following is hacked/copied from Ubiquity
rpath = os.path.join(dirpath, name)
sourcepath = os.path.join(SOURCE, rpath)
targetpath = os.path.join(DEST, rpath)
st = os.lstat(sourcepath)
mode = stat.S_IMODE(st.st_mode)
# now show the world what we're doing
our_current += 1
self.update_progress(total=our_total,
current=our_current,
message=_("Copying %s") % rpath)
if os.path.exists(targetpath):
if not os.path.isdir(targetpath):
os.remove(targetpath)
if stat.S_ISLNK(st.st_mode):
if os.path.lexists(targetpath):
os.unlink(targetpath)
linkto = os.readlink(sourcepath)
os.symlink(linkto, targetpath)
elif stat.S_ISDIR(st.st_mode):
if not os.path.isdir(targetpath):
os.mkdir(targetpath, mode)
elif stat.S_ISCHR(st.st_mode):
os.mknod(targetpath, stat.S_IFCHR | mode, st.st_rdev)
elif stat.S_ISBLK(st.st_mode):
os.mknod(targetpath, stat.S_IFBLK | mode, st.st_rdev)
elif stat.S_ISFIFO(st.st_mode):
os.mknod(targetpath, stat.S_IFIFO | mode)
elif stat.S_ISSOCK(st.st_mode):
os.mknod(targetpath, stat.S_IFSOCK | mode)
elif stat.S_ISREG(st.st_mode):
# we don't do blacklisting yet..
try:
os.unlink(targetpath)
except:
pass
self.do_copy_file(sourcepath, targetpath)
os.lchown(targetpath, st.st_uid, st.st_gid)
if not stat.S_ISLNK(st.st_mode):
os.chmod(targetpath, mode)
if stat.S_ISDIR(st.st_mode):
directory_times.append(
(targetpath, st.st_atime, st.st_mtime))
# os.utime() sets timestamp of target, not link
elif not stat.S_ISLNK(st.st_mode):
os.utime(targetpath, (st.st_atime, st.st_mtime))
# Apply timestamps to all directories now that the items within them
# have been copied.
print " --> Restoring meta-info"
for dirtime in directory_times:
(directory, atime, mtime) = dirtime
try:
self.update_progress(
pulse=True,
message=_("Restoring meta-information on %s") % directory)
os.utime(directory, (atime, mtime))
except OSError:
pass
# Steps:
our_total = 11
our_current = 0
# chroot
print " --> Chrooting"
self.update_progress(total=our_total,
current=our_current,
message=_("Entering the system ..."))
os.system("mount --bind /dev/ /target/dev/")
os.system("mount --bind /dev/shm /target/dev/shm")
os.system("mount --bind /dev/pts /target/dev/pts")
os.system("mount --bind /sys/ /target/sys/")
os.system("mount --bind /proc/ /target/proc/")
os.system("mv /target/etc/resolv.conf /target/etc/resolv.conf.bk")
os.system("cp -f /etc/resolv.conf /target/etc/resolv.conf")
kernelversion = commands.getoutput("uname -r")
os.system(
"cp /lib/live/mount/medium/live/vmlinuz /target/boot/vmlinuz-%s" %
kernelversion)
found_initrd = False
for initrd in [
"/lib/live/mount/medium/live/initrd.img",
"/lib/live/mount/medium/live/initrd.lz"
]:
if os.path.exists(initrd):
os.system("cp %s /target/boot/initrd.img-%s" %
(initrd, kernelversion))
found_initrd = True
break
if not found_initrd:
print "WARNING: No initrd found!!"
if (setup.gptonefi):
os.system("mkdir -p /target/boot/efi/EFI/linuxmint")
os.system(
"cp /lib/live/mount/medium/EFI/BOOT/grubx64.efi /target/boot/efi/EFI/linuxmint"
)
os.system("mkdir -p /target/debs")
os.system(
"cp /lib/live/mount/medium/pool/main/g/grub2/grub-efi* /target/debs/"
)
os.system(
"cp /lib/live/mount/medium/pool/main/e/efibootmgr/efibootmgr* /target/debs/"
)
os.system(
"cp /lib/live/mount/medium/pool/main/e/efivar/* /target/debs/")
self.do_run_in_chroot("dpkg -i /debs/*")
os.system("rm -rf /target/debs")
# Detect cdrom device
# TODO : properly detect cdrom device
# Mount it
# os.system("mkdir -p /target/media/cdrom")
# if (int(os.system("mount /dev/sr0 /target/media/cdrom"))):
# print " --> Failed to mount CDROM. Install will fail"
# self.do_run_in_chroot("apt-cdrom -o Acquire::cdrom::AutoDetect=false -m add")
# remove live-packages (or w/e)
print " --> Removing live packages"
our_current += 1
self.update_progress(
total=our_total,
current=our_current,
message=_("Removing live configuration (packages)"))
with open("/lib/live/mount/medium/live/filesystem.packages-remove",
"r") as fd:
line = fd.read().replace('\n', ' ')
self.do_run_in_chroot("apt-get remove --purge --yes --force-yes %s" %
line)
# add new user
print " --> Adding new user"
our_current += 1
self.update_progress(total=our_total,
current=our_current,
message=_("Adding new user to the system"))
self.do_run_in_chroot(
'adduser --disabled-login --gecos "{real_name}" {username}'.format(
real_name=setup.real_name.replace('"', r'\"'),
username=setup.username))
for group in 'adm audio bluetooth cdrom dialout dip fax floppy fuse lpadmin netdev plugdev powerdev sambashare scanner sudo tape users vboxusers video'.split(
):
self.do_run_in_chroot("adduser {user} {group}".format(
user=setup.username, group=group))
fp = open("/target/tmp/.passwd", "w")
fp.write(setup.username + ":" + setup.password1 + "\n")
fp.write("root:" + setup.password1 + "\n")
fp.close()
self.do_run_in_chroot("cat /tmp/.passwd | chpasswd")
os.system("rm -f /target/tmp/.passwd")
# Set autologin for user if they so elected
if setup.autologin:
# LightDM
self.do_run_in_chroot(
r"sed -i -r 's/^#?(autologin-user)\s*=.*/\1={user}/' /etc/lightdm/lightdm.conf"
.format(user=setup.username))
# MDM
self.do_run_in_chroot(
r"sed -i -r -e '/^AutomaticLogin(Enable)?\s*=/d' -e 's/^(\[daemon\])/\1\nAutomaticLoginEnable=true\nAutomaticLogin={user}/' /etc/mdm/mdm.conf"
.format(user=setup.username))
# GDM3
self.do_run_in_chroot(
r"sed -i -r -e '/^(#\s*)?AutomaticLogin(Enable)?\s*=/d' -e 's/^(\[daemon\])/\1\nAutomaticLoginEnable=true\nAutomaticLogin={user}/' /etc/gdm3/daemon.conf"
.format(user=setup.username))
# KDE4
self.do_run_in_chroot(
r"sed -i -r -e 's/^#?(AutomaticLoginEnable)\s*=.*/\1=true/' -e 's/^#?(AutomaticLoginUser)\s*.*/\1={user}/' /etc/kde4/kdm/kdmrc"
.format(user=setup.username))
# LXDM
self.do_run_in_chroot(
r"sed -i -r -e 's/^#?(autologin)\s*=.*/\1={user}/' /etc/lxdm/lxdm.conf"
.format(user=setup.username))
# SLiM
self.do_run_in_chroot(
r"sed -i -r -e 's/^#?(default_user)\s.*/\1 {user}/' -e 's/^#?(auto_login)\s.*/\1 yes/' /etc/slim.conf"
.format(user=setup.username))
# Add user's face
os.system("cp /tmp/live-installer-face.png /target/home/%s/.face" %
setup.username)
self.do_run_in_chroot("chown %s:%s /home/%s/.face" %
(setup.username, setup.username, setup.username))
# Make the new user the default user in KDM
if os.path.exists('/target/etc/kde4/kdm/kdmrc'):
defUsrCmd = "sed -i 's/^#DefaultUser=.*/DefaultUser="******"/g' " + kdmrcPath
print defUsrCmd
os.system(defUsrCmd)
# write the /etc/fstab
print " --> Writing fstab"
our_current += 1
self.update_progress(
total=our_total,
current=our_current,
message=_("Writing filesystem mount information to /etc/fstab"))
# make sure fstab has default /proc and /sys entries
if (not os.path.exists("/target/etc/fstab")):
os.system(
"echo \"#### Static Filesystem Table File\" > /target/etc/fstab"
)
fstab = open("/target/etc/fstab", "a")
fstab.write("proc\t/proc\tproc\tdefaults\t0\t0\n")
if (not setup.skip_mount):
for partition in setup.partitions:
if (partition.mount_as is not None and partition.mount_as != ""
and partition.mount_as != "None"):
partition_uuid = partition.partition.path # If we can't find the UUID we use the path
blkid = commands.getoutput('blkid').split('\n')
for blkid_line in blkid:
blkid_elements = blkid_line.split(':')
if blkid_elements[0] == partition.partition.path:
blkid_mini_elements = blkid_line.split()
for blkid_mini_element in blkid_mini_elements:
if "UUID=" in blkid_mini_element:
partition_uuid = blkid_mini_element.replace(
'"', '').strip()
break
break
fstab.write("# %s\n" % (partition.partition.path))
if (partition.mount_as == "/"):
fstab_fsck_option = "1"
else:
fstab_fsck_option = "0"
if ("ext" in partition.type):
fstab_mount_options = "rw,errors=remount-ro"
else:
fstab_mount_options = "defaults"
if partition.type == "fat16" or partition.type == "fat32":
fs = "vfat"
else:
fs = partition.type
if (fs == "swap"):
fstab.write("%s\tswap\tswap\tsw\t0\t0\n" %
partition_uuid)
else:
fstab.write(
"%s\t%s\t%s\t%s\t%s\t%s\n" %
(partition_uuid, partition.mount_as, fs,
fstab_mount_options, "0", fstab_fsck_option))
fstab.close()
def is_socket(self) -> bool:
"""True if file is a socket, otherwise False."""
e = self._get_unix_extension()
if e is not None:
return stat.S_ISSOCK(e)
return False
def _make_server_creators(handler, *, loop, ssl_context, host, port, path,
sock, backlog):
scheme = 'https' if ssl_context else 'http'
base_url = URL.build(scheme=scheme, host='localhost', port=port)
if path is None:
paths = ()
elif isinstance(path, (str, bytes, bytearray, memoryview))\
or not isinstance(path, Iterable):
paths = (path, )
else:
paths = path
if sock is None:
socks = ()
elif not isinstance(sock, Iterable):
socks = (sock, )
else:
socks = sock
if host is None:
if (paths or socks) and not port:
hosts = ()
else:
hosts = ("0.0.0.0", )
elif isinstance(host, (str, bytes, bytearray, memoryview))\
or not isinstance(host, Iterable):
hosts = (host, )
else:
hosts = host
if hosts and port is None:
port = 8443 if ssl_context else 8080
server_creations = []
uris = [str(base_url.with_host(host).with_port(port)) for host in hosts]
if hosts:
# Multiple hosts bound to same server is available in most loop
# implementations, but only send multiple if we have multiple.
host_binding = hosts[0] if len(hosts) == 1 else hosts
server_creations.append(
loop.create_server(handler,
host_binding,
port,
ssl=ssl_context,
backlog=backlog))
for path in paths:
# Most loop implementations don't support multiple paths bound in same
# server, so create a server for each.
server_creations.append(
loop.create_unix_server(handler,
path,
ssl=ssl_context,
backlog=backlog))
uris.append('{}://unix:{}:'.format(scheme, path))
# Clean up prior socket path if stale and not abstract.
# CPython 3.5.3+'s event loop already does this. See
# https://github.com/python/asyncio/issues/425
if path[0] not in (0, '\x00'): # pragma: no branch
try:
if stat.S_ISSOCK(os.stat(path).st_mode):
os.remove(path)
except FileNotFoundError:
pass
for sock in socks:
server_creations.append(
loop.create_server(handler,
sock=sock,
ssl=ssl_context,
backlog=backlog))
if hasattr(socket, 'AF_UNIX') and sock.family == socket.AF_UNIX:
uris.append('{}://unix:{}:'.format(scheme, sock.getsockname()))
else:
host, port = sock.getsockname()[:2]
uris.append(str(base_url.with_host(host).with_port(port)))
return server_creations, uris
def main():
# If we're called from xinetd, set idle to non-zero
mode = os.fstat(sys.stdin.fileno()).st_mode
if stat.S_ISSOCK(mode):
OPTIONS["idle"] = 30
OPTIONS["logger"] = "syslog"
# setup option parsing
opt_help = dict(
port=dict(type="int", help="The port to bind to for new requests"),
idle=dict(type="int", help="How long to wait for input"),
timeout=dict(type="int", help="How long to wait for a given request"),
max_blksize=dict(type="int", help="The maximum block size to permit"),
prefix=dict(type="string", help="Where files are stored by default [" + OPTIONS["prefix"] + "]"),
logger=dict(type="string", help="How to log"),
file_cmd=dict(type="string", help="The location of the 'file' command"),
user=dict(type="string", help="The user to run as [nobody]"),
)
parser = optparse.OptionParser(
formatter=optparse.IndentedHelpFormatter(),
usage=globals()['__doc__'],
version=VERSION)
parser.add_option('-v', '--verbose', action='store_true', default=False,
help="Increase output verbosity")
parser.add_option('-d', '--debug', action='store_true', default=False,
help="Debug (vastly increases output verbosity)")
parser.add_option('-c', '--cache', action='store_true', default=True,
help="Use a cache to help find hosts w/o IP address")
parser.add_option('--cache-time', action='store', type="int", default=5 * 60,
help="How long an ip->name mapping is valid")
parser.add_option('--neg-cache-time', action='store', type="int", default=10,
help="How long an ip->name mapping is valid")
opts = list(opt_help.keys())
opts.sort()
for k in opts:
v = opt_help[k]
parser.add_option("--" + k, default=OPTIONS[k], type=v["type"], help=v["help"])
parser.add_option('-B', dest="max_blksize", type="int", default=1428,
help="alias for --max-blksize, for in.tftpd compatibility")
# Actually read the args
(options, args) = parser.parse_args()
for attr in dir(options):
if attr in OPTIONS:
OPTIONS[attr] = getattr(options, attr)
if stat.S_ISSOCK(mode) or OPTIONS["logger"] == "syslog":
# log to syslog. Facility 11 isn't in the class, but it's FTP on linux
logger = logging.handlers.SysLogHandler("/dev/log", 11)
logger.setFormatter(
logging.Formatter('%(filename)s: %(levelname)s: %(message)s'))
elif OPTIONS["logger"] == "stream":
# log to stdout
logger = logging.StreamHandler()
logger.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
else:
logger = logging.FileHandler("/var/log/tftpd")
logger.setFormatter(logging.Formatter(
"%(asctime)s %(name)s: %(levelname)s: %(message)s"))
logging.getLogger().addHandler(logger)
if OPTIONS["debug"]:
logging.getLogger().setLevel(logging.DEBUG)
elif OPTIONS["verbose"]:
logging.getLogger().setLevel(logging.INFO)
else:
logging.getLogger().setLevel(logging.WARN)
if stat.S_ISSOCK(mode):
OPTIONS["sock"] = socket.fromfd(sys.stdin.fileno(), socket.AF_INET, socket.SOCK_DGRAM, 0)
else:
OPTIONS["sock"] = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
OPTIONS["sock"].setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
OPTIONS["sock"].bind(("", OPTIONS["port"]))
except socket.error as e:
if e[0] in (errno.EPERM, errno.EACCES):
print("Unable to bind to port %d" % OPTIONS["port"])
return -1
else:
raise
OPTIONS["sock"].setblocking(0)
if os.getuid() == 0:
uid = pwd.getpwnam(OPTIONS["user"])[2]
os.setreuid(uid, uid)
# This takes a while, so do it after we open the port, so we
# don't drop the packet that spawned us
templar = cobbler.templar.Templar(None)
io_loop = ioloop.IOLoop.instance()
io_loop.add_handler(OPTIONS["sock"].fileno(), partial(new_req, OPTIONS["sock"], templar), io_loop.READ)
# Shove the timeout into OPTIONS, because it's there
if OPTIONS["idle"] > 0:
OPTIONS["idle_timer"] = io_loop.add_timeout(time.time() + OPTIONS["idle"], lambda: idle_out())
logging.info('Starting Eventloop')
try:
try:
io_loop.start()
except KeyboardInterrupt:
# Someone hit ^C
logging.info('Exiting')
finally:
OPTIONS["sock"].close()
return 0
