def mount_old(self):
self.mount_process = subprocess.Popen(
[
os.path.join(self.basedir_old, "bin", "mount.s3ql"),
"--fg",
"--cachedir",
self.cache_dir,
"--log",
"none",
"--quiet",
"--authfile",
"/dev/null",
"--compress",
"zlib",
self.storage_url,
self.mnt_dir,
],
stdin=subprocess.PIPE,
universal_newlines=True,
)
if self.backend_login is not None:
print(self.backend_login, file=self.mount_process.stdin)
print(self.backend_passphrase, file=self.mount_process.stdin)
if self.passphrase is not None:
print(self.passphrase, file=self.mount_process.stdin)
self.mount_process.stdin.close()
def poll():
if os.path.ismount(self.mnt_dir):
return True
assert self.mount_process.poll() is None
retry(30, poll)
def test_cp_inode_invalidate(self):
if os.getuid() != 0:
pytest.skip('test_cp_inode_invalidate requires root, skipping.')
self.passphrase = None
self.mkfs()
# Run monkeypatched mount.s3ql with overriden pyfuse3.invalidate_inode :
# Drop kernel dentries and inodes cache just before calling pyfuse3.invalidate_inode
cmd = ([sys.executable, os.path.join(os.path.dirname(__file__), 'mount_helper.py'), "--fg", '--cachedir',
self.cache_dir, '--log', 'none',
'--compress', 'zlib', '--quiet', self.storage_url, self.mnt_dir,
'--authfile', '/dev/null' ])
self.mount_process = subprocess.Popen(cmd, universal_newlines=True)
def poll():
if os.path.ismount(self.mnt_dir):
return True
assert self.mount_process.poll() is None
retry(10, poll)
os.mkdir(os.path.join(self.mnt_dir, 'orig'))
cmd = (self.s3ql_cmd_argv('s3qlcp') +
[ '--quiet', os.path.join(self.mnt_dir, 'orig'),
os.path.join(self.mnt_dir, 'copy')])
cp_process = subprocess.Popen(cmd)
retry(5, lambda : cp_process.poll() is not None)
assert cp_process.wait() == 0
self.umount()
self.fsck()
def mount(self, expect_fail=None, extra_args=[]):
cmd = (self.s3ql_cmd_argv('mount.s3ql') + [
"--fg", '--cachedir', self.cache_dir, '--log', 'none',
'--compress', 'zlib', '--quiet', self.storage_url, self.mnt_dir,
'--authfile', '/dev/null'
] + extra_args)
self.mount_process = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
universal_newlines=True)
if self.backend_login is not None:
print(self.backend_login, file=self.mount_process.stdin)
print(self.backend_passphrase, file=self.mount_process.stdin)
if self.passphrase is not None:
print(self.passphrase, file=self.mount_process.stdin)
self.mount_process.stdin.close()
if expect_fail:
retry(30, self.mount_process.poll)
assert self.mount_process.returncode == expect_fail
else:
def poll():
if os.path.ismount(self.mnt_dir):
return True
assert self.mount_process.poll() is None
retry(30, poll)
def mount(self, fatal_warnings=True, expect_fail=None):
cmd = (self.s3ql_cmd_argv('mount.s3ql') +
["--fg", '--cachedir', self.cache_dir, '--log', 'none',
'--compress', 'zlib', '--quiet', self.storage_url, self.mnt_dir,
'--authfile', '/dev/null' ])
if fatal_warnings:
cmd.append('--fatal-warnings')
self.mount_process = subprocess.Popen(cmd, stdin=subprocess.PIPE,
universal_newlines=True)
if self.backend_login is not None:
print(self.backend_login, file=self.mount_process.stdin)
print(self.backend_passphrase, file=self.mount_process.stdin)
if self.passphrase is not None:
print(self.passphrase, file=self.mount_process.stdin)
self.mount_process.stdin.close()
if expect_fail:
retry(30, self.mount_process.poll)
assert self.mount_process.returncode == expect_fail
else:
def poll():
if os.path.ismount(self.mnt_dir):
return True
assert self.mount_process.poll() is None
retry(30, poll)
def startstop(srvs, action, count, interval):
srv_baseurl = CLUSTER_URL_PTR % (self.url,
self.cluster_name) + '/services/'
state = 'INSTALLED' if action == 'stop' else 'STARTED'
for srv in srvs:
srv_url = srv_baseurl + srv
config = {
'RequestInfo': {
'context': '%s %s services' % (action, srv)
},
'ServiceInfo': {
'state': state
}
}
rc = self.p.put(srv_url, config)
# check startstop status
if rc:
get_stat = lambda: self.p.get(srv_url)['ServiceInfo'][
'state'] == state
retry(get_stat, count, interval,
'HDP service %s %s' % (srv, action))
else:
if action == 'stop': action += 'p'
info('HDP service %s had already been %sed' %
(srv, action))
def umount(self):
with open("/dev/null", "wb") as devnull:
retry(5, lambda: subprocess.call(["fuser", "-m", self.mnt_dir], stdout=devnull, stderr=devnull) == 1)
proc = subprocess.Popen(self.s3ql_cmd_argv("umount.s3ql") + ["--quiet", self.mnt_dir])
retry(90, lambda: proc.poll() is not None)
assert proc.wait() == 0
assert self.mount_process.poll() == 0
assert not os.path.ismount(self.mnt_dir)
def umount_old(self):
with open('/dev/null', 'wb') as devnull:
retry(5, lambda: subprocess.call(['fuser', '-m', self.mnt_dir],
stdout=devnull, stderr=devnull) == 1)
proc = subprocess.Popen([os.path.join(self.basedir_old, 'bin', 'umount.s3ql'),
'--quiet', self.mnt_dir])
retry(90, lambda : proc.poll() is not None)
self.assertEqual(proc.wait(), 0)
self.assertEqual(self.mount_process.poll(), 0)
self.assertFalse(os.path.ismount(self.mnt_dir))
def umount_old(self):
with open('/dev/null', 'wb') as devnull:
retry(5, lambda: subprocess.call(['fuser', '-m', self.mnt_dir],
stdout=devnull, stderr=devnull) == 1)
proc = subprocess.Popen([os.path.join(self.basedir_old, 'bin', 'umount.s3ql'),
'--quiet', self.mnt_dir])
retry(90, lambda : proc.poll() is not None)
assert proc.wait() == 0
assert self.mount_process.poll() == 0
assert not os.path.ismount(self.mnt_dir)
def umount(self):
with open('/dev/null', 'wb') as devnull:
retry(5, lambda: subprocess.call(['fuser', '-m', self.mnt_dir],
stdout=devnull, stderr=devnull) == 1)
proc = subprocess.Popen(self.s3ql_cmd_argv('umount.s3ql') +
['--quiet', self.mnt_dir])
retry(90, lambda : proc.poll() is not None)
assert proc.wait() == 0
assert self.mount_process.poll() == 0
assert not os.path.ismount(self.mnt_dir)
def umount(self):
with open('/dev/null', 'wb') as devnull:
retry(5, lambda: subprocess.call(['fuser', '-m', self.mnt_dir],
stdout=devnull, stderr=devnull) == 1)
proc = subprocess.Popen(self.s3ql_cmd_argv('umount.s3ql') +
['--quiet', self.mnt_dir])
retry(90, lambda : proc.poll() is not None)
self.assertEqual(proc.wait(), 0)
self.assertEqual(self.mount_process.poll(), 0)
self.assertFalse(os.path.ismount(self.mnt_dir))
def teardown_method(self, method):
self.umount_fuse()
os.rmdir(self.mnt_dir)
# Give mount process a little while to terminate
if self.mount_process is not None:
try:
retry(90, lambda : self.mount_process.poll() is not None)
except TimeoutError:
# Ignore errors during teardown
pass
shutil.rmtree(self.cache_dir)
shutil.rmtree(self.backend_dir)
def teardown_method(self, method):
self.umount_fuse()
os.rmdir(self.mnt_dir)
# Give mount process a little while to terminate
if self.mount_process is not None:
try:
retry(90, lambda: self.mount_process.poll() is not None)
except TimeoutError:
# Ignore errors during teardown
pass
shutil.rmtree(self.cache_dir)
shutil.rmtree(self.backend_dir)
def startstop(srvs, action, count, interval):
srv_baseurl = CLUSTER_URL_PTR % (self.url, self.cluster_name) + '/services/'
state = 'INSTALLED' if action == 'stop' else 'STARTED'
for srv in srvs:
srv_url = srv_baseurl + srv
config = {'RequestInfo': {'context' :'%s %s services' % (action, srv)}, 'ServiceInfo': {'state' : state}}
rc = self.p.put(srv_url, config)
# check startstop status
if rc:
get_stat = lambda: self.p.get(srv_url)['ServiceInfo']['state'] == state
retry(get_stat, count, interval, 'HDP service %s %s' % (srv, action))
else:
if action == 'stop': action += 'p'
info('HDP service %s had already been %sed' % (srv, action))
def tst_bug382(self):
dirname = self.newname()
fullname = self.mnt_dir + "/" + dirname
os.mkdir(fullname)
assert stat.S_ISDIR(os.stat(fullname).st_mode)
assert dirname in pyfuse3.listdir(self.mnt_dir)
cmd = ('(%d, %r)' % (pyfuse3.ROOT_INODE, path2bytes(dirname))).encode()
pyfuse3.setxattr('%s/%s' % (self.mnt_dir, CTRL_NAME), 'rmtree', cmd)
# Invalidation is asynchronous...
try:
retry(5, lambda: not os.path.exists(fullname))
except RetryTimeoutError:
pass # assert_raises should fail
assert_raises(FileNotFoundError, os.stat, fullname)
assert dirname not in pyfuse3.listdir(self.mnt_dir)
def _logout(self, sql, client):
# Client logout
# Use datetime.fromtimestamp() with int(time()) to have microsecond=0
logout_before = datetime_before()
client.stop()
for when in retry(timeout=QUERY_TIMEOUT):
# Get last MySQL row
cursor = self.query(sql)
# Check number of rows
if not cursor.rowcount:
continue
self.assertEqual(cursor.rowcount, 1)
# Read row columns
(ip_saddr, user_id, username, os_sysname, os_release, os_version,
end_time) = self.fetchone(cursor)
if not end_time:
continue
break
# Check values
if not POSTGRESQL:
# FIXME: Convert string to datetime for PostgreSQL
logout_after = datetime_after()
self.assert_(logout_before <= end_time <= logout_after)
def _login(self, sql):
# Client login
client = self.user.createClientWithCerts()
self.assert_(connectClient(client))
# Check number of rows
for when in retry(timeout=QUERY_TIMEOUT):
cursor = self.query(sql)
for line in self.nuauth.readlines():
pass
if cursor.rowcount:
break
self.assertEqual(cursor.rowcount, 1)
# Read row columns
(ip_saddr, user_id, username, os_sysname, os_release, os_version,
end_time) = self.fetchone(cursor)
if not POSTGRESQL:
ip_saddr = ntohl(ip_saddr) & 0xFFFFFFFF
# Check values
self.assertEqual(IP(ip_saddr), client.ip)
self.assertEqual(user_id, self.user.uid)
self.assertEqual(username, client.username)
self.assertEqual(os_sysname, OS_SYSNAME)
self.assertEqual(os_release, OS_RELEASE)
self.assertEqual(os_version, OS_VERSION)
return client
def _logout(self, sql, client):
# Client logout
# Use datetime.fromtimestamp() with int(time()) to have microsecond=0
logout_before = datetime_before()
client.stop()
for when in retry(timeout=QUERY_TIMEOUT):
# Get last MySQL row
cursor = self.query(sql)
# Check number of rows
if not cursor.rowcount:
continue
self.assertEqual(cursor.rowcount, 1)
# Read row columns
(ip_saddr, user_id, username, os_sysname,
os_release, os_version, end_time) = self.fetchone(cursor)
if not end_time:
continue
break
# Check values
if not POSTGRESQL:
# FIXME: Convert string to datetime for PostgreSQL
logout_after = datetime_after()
self.assert_(logout_before <= end_time <= logout_after)
def _login(self, sql):
# Client login
client = self.user.createClientWithCerts()
self.assert_(connectClient(client))
# Check number of rows
for when in retry(timeout=QUERY_TIMEOUT):
cursor = self.query(sql)
for line in self.nuauth.readlines():
pass
if cursor.rowcount:
break
self.assertEqual(cursor.rowcount, 1)
# Read row columns
(ip_saddr, user_id, username, os_sysname,
os_release, os_version, end_time) = self.fetchone(cursor)
if not POSTGRESQL:
ip_saddr = ntohl(ip_saddr) & 0xFFFFFFFF
# Check values
self.assertEqual(IP(ip_saddr), client.ip)
self.assertEqual(user_id, self.user.uid)
self.assertEqual(username, client.username)
self.assertEqual(os_sysname, OS_SYSNAME)
self.assertEqual(os_release, OS_RELEASE)
self.assertEqual(os_version, OS_VERSION)
return client
def mount_old(self):
self.mount_process = subprocess.Popen([os.path.join(self.basedir_old, 'bin', 'mount.s3ql'),
"--fg", '--cachedir', self.cache_dir, '--log',
'none', '--quiet', '--authfile', '/dev/null',
'--compress', 'zlib', self.storage_url, self.mnt_dir],
stdin=subprocess.PIPE, universal_newlines=True)
if self.backend_login is not None:
print(self.backend_login, file=self.mount_process.stdin)
print(self.backend_passphrase, file=self.mount_process.stdin)
print(self.passphrase, file=self.mount_process.stdin)
self.mount_process.stdin.close()
def poll():
if os.path.ismount(self.mnt_dir):
return True
self.assertIsNone(self.mount_process.poll())
retry(30, poll)
def get_logs(self, from_id):
key = self.user_key or self.service_key
data = common.retry(
self.get_query,
const.LINK +
'query=flightlogs&search=id&readaccesskey={}&fromid={}'.format(
key, from_id),
error_type=TooManyConnectionsException)
logs = pd.DataFrame.from_csv(StringIO(data))
logs = logs[(logs.MakeModel != 'Airbus A321')
& (logs.MakeModel != 'Boeing 737-800') &
(logs.Type == 'flight')]
logs['Distance'] = logs.apply(
lambda x, self=self: self.get_distance(x['From'], x['To']), axis=1)
logs = pd.merge(logs,
self.aircrafts,
left_on='MakeModel',
right_on='Model')
logs['FlightTimeH'] = logs.apply(
lambda x: int(x['FlightTime'].split(':')[0]), axis=1)
logs['FlightTimeM'] = logs.apply(
lambda x: int(x['FlightTime'].split(':')[1]), axis=1)
logs = logs[(logs.FlightTimeH > 0) | (logs.FlightTimeM > 0)]
logs = logs[logs.Distance > 0]
logs['AvSpeed'] = logs.apply(
lambda x: 60 * x['Distance'] /
(60 * x['FlightTimeH'] + x['FlightTimeM']),
axis=1)
def testFilter(self):
"""
User logs in, opens an authenticated connection, and
closes the connection. Make sure that MySQL records the connection,
only once, with the right parameters.
"""
client = self.user.createClientWithCerts()
time_before = int(time())
timestamp_before = datetime_before()
# Open allowed port
testAllowPort(self, self.iptables, client)
# Query DB
if not POSTGRESQL:
timestamp_field = "timestamp, "
else:
timestamp_field = ""
sql = \
"SELECT username, user_id, client_os, client_app, " \
"tcp_dport, ip_saddr, ip_daddr, oob_time_sec, ip_protocol, " \
"%sstart_timestamp, end_timestamp, oob_prefix " \
"FROM %s WHERE oob_time_sec >= %s AND state=1;" \
% (timestamp_field, DB_PACKET_TABLE, time_before)
# Do the query
for when in retry(timeout=QUERY_TIMEOUT):
cursor = self.query(sql)
if cursor.rowcount:
break
# Read result
row = self.fetchone(cursor)
timestamp_after = datetime_after()
self.assertEqual(cursor.rowcount, 1)
if POSTGRESQL:
(username, user_id, client_os, client_app, tcp_dport, ip_saddr,
ip_daddr, oob_time_sec, ip_protocol, start_timestamp,
end_timestamp, oob_prefix) = row
else:
(username, user_id, client_os, client_app, tcp_dport, ip_saddr,
ip_daddr, oob_time_sec, ip_protocol, timestamp, start_timestamp,
end_timestamp, oob_prefix) = row
# Check values
self.assertEqual(username, client.username)
self.assertEqual(user_id, self.user.uid)
self.assertEqual(client_os, CLIENT_OS)
self.assertEqual(client_app, CLIENT_APP)
self.assertEqual(tcp_dport, VALID_PORT)
self.assertEqual(IP(ip_saddr), client.ip)
self.assert_(timestamp_before <= datetime.fromtimestamp(oob_time_sec)
<= timestamp_after)
if not POSTGRESQL:
self.assert_(timestamp
and timestamp_before <= timestamp <= timestamp_after)
self.assertEqual(ip_protocol, 6)
self.assertEqual(oob_prefix, OOB_PREFIX)
def teardown_method(self, method):
self.umount_fuse()
os.rmdir(self.mnt_dir)
# Give mount process a little while to terminate
if self.mount_process is not None:
try:
retry(10, lambda: self.mount_process.poll() is not None)
except TimeoutError:
self.mount_process.terminate()
try:
self.mount_process.wait(1)
except subprocess.TimeoutExpired:
self.mount_process.kill()
shutil.rmtree(self.cache_dir)
shutil.rmtree(self.backend_dir)
def mount_old(self):
self.mount_process = subprocess.Popen([os.path.join(self.basedir_old, 'bin', 'mount.s3ql'),
"--fg", '--cachedir', self.cache_dir, '--log',
'none', '--quiet', '--authfile', '/dev/null',
'--compress', 'zlib', self.storage_url, self.mnt_dir],
stdin=subprocess.PIPE, universal_newlines=True)
if self.backend_login is not None:
print(self.backend_login, file=self.mount_process.stdin)
print(self.backend_passphrase, file=self.mount_process.stdin)
if self.passphrase is not None:
print(self.passphrase, file=self.mount_process.stdin)
self.mount_process.stdin.close()
def poll():
if os.path.ismount(self.mnt_dir):
return True
assert self.mount_process.poll() is None
retry(30, poll)
def testFilter(self):
"""
User logs in, opens an authenticated connection, and
closes the connection. Make sure that MySQL records the connection,
only once, with the right parameters.
"""
client = self.user.createClientWithCerts()
time_before = int(time())
timestamp_before = datetime_before()
# Open allowed port
testAllowPort(self, self.iptables, client)
# Query DB
if not POSTGRESQL:
timestamp_field = "timestamp, "
else:
timestamp_field = ""
sql = \
"SELECT username, user_id, client_os, client_app, " \
"tcp_dport, ip_saddr, ip_daddr, oob_time_sec, ip_protocol, " \
"%sstart_timestamp, end_timestamp, oob_prefix " \
"FROM %s WHERE oob_time_sec >= %s AND state=1;" \
% (timestamp_field, DB_PACKET_TABLE, time_before)
# Do the query
for when in retry(timeout=QUERY_TIMEOUT):
cursor = self.query(sql)
if cursor.rowcount:
break
# Read result
row = self.fetchone(cursor)
timestamp_after = datetime_after()
self.assertEqual(cursor.rowcount, 1)
if POSTGRESQL:
(username, user_id, client_os, client_app,
tcp_dport, ip_saddr, ip_daddr, oob_time_sec, ip_protocol,
start_timestamp, end_timestamp, oob_prefix) = row
else:
(username, user_id, client_os, client_app,
tcp_dport, ip_saddr, ip_daddr, oob_time_sec, ip_protocol,
timestamp, start_timestamp, end_timestamp, oob_prefix) = row
# Check values
self.assertEqual(username, client.username)
self.assertEqual(user_id, self.user.uid)
self.assertEqual(client_os, CLIENT_OS)
self.assertEqual(client_app, CLIENT_APP)
self.assertEqual(tcp_dport, VALID_PORT)
self.assertEqual(IP(ip_saddr), client.ip)
self.assert_(timestamp_before <= datetime.fromtimestamp(oob_time_sec) <= timestamp_after)
if not POSTGRESQL:
self.assert_(timestamp and timestamp_before <= timestamp <= timestamp_after)
self.assertEqual(ip_protocol, 6)
self.assertEqual(oob_prefix, OOB_PREFIX)
def teardown_method(self, method):
with open("/dev/null", "wb") as devnull:
if platform.system() == "Darwin":
subprocess.call(["umount", "-l", self.mnt_dir], stderr=devnull)
else:
subprocess.call(["fusermount", "-z", "-u", self.mnt_dir], stderr=devnull)
os.rmdir(self.mnt_dir)
# Give mount process a little while to terminate
if self.mount_process is not None:
try:
retry(90, lambda: self.mount_process.poll() is not None)
except TimeoutError:
# Ignore errors during teardown
pass
shutil.rmtree(self.cache_dir)
shutil.rmtree(self.backend_dir)
def get_aircrafts_by_icao(self, icao):
data = common.retry(self.get_query,
const.LINK +
'query=icao&search=aircraft&icao={}'.format(icao),
error_type=TooManyConnectionsException)
aircrafts = pd.DataFrame.from_csv(StringIO(data))
aircrafts.RentalDry = aircrafts.RentalDry.astype(float)
aircrafts.RentalWet = aircrafts.RentalWet.astype(float)
return aircrafts
def __init__(self, title, url, user_name, password):
"""
:param title: str - title of the article
:param url: str - url of the wordpress web
:param user_name: str
:param password: str
"""
self.client = retry(Client, url, user_name, password)
self.title = title
def __init__(self, title, url, user_name, password):
self.fieldnames_list = ["Picture", "Name", "Rating", "Price"]
self.client = retry(Client, url, user_name, password)
self.title = title
self.article_dir = path.join(self.ARTICLES_DIR, title)
if not path.isdir(self.ARTICLES_DIR):
makedirs(self.article_dir)
if not path.isdir(self.article_dir):
mkdir(self.article_dir)
def tearDown(self):
with open('/dev/null', 'wb') as devnull:
if platform.system() == 'Darwin':
subprocess.call(['umount', '-l', self.mnt_dir], stderr=devnull)
else:
subprocess.call(['fusermount', '-z', '-u', self.mnt_dir],
stderr=devnull)
os.rmdir(self.mnt_dir)
# Give mount process a little while to terminate
if self.mount_process is not None:
try:
retry(90, lambda : self.mount_process.poll() is not None)
except TimeoutError:
# Ignore errors during teardown
pass
shutil.rmtree(self.cache_dir)
shutil.rmtree(self.backend_dir)
def mount(self, fatal_warnings=True):
cmd = (self.s3ql_cmd_argv('mount.s3ql') +
["--fg", '--cachedir', self.cache_dir, '--log', 'none',
'--compress', 'zlib', '--quiet', self.storage_url, self.mnt_dir,
'--authfile', '/dev/null' ])
if fatal_warnings:
cmd.append('--fatal-warnings')
self.mount_process = subprocess.Popen(cmd, stdin=subprocess.PIPE,
universal_newlines=True)
if self.backend_login is not None:
print(self.backend_login, file=self.mount_process.stdin)
print(self.backend_passphrase, file=self.mount_process.stdin)
print(self.passphrase, file=self.mount_process.stdin)
self.mount_process.stdin.close()
def poll():
if os.path.ismount(self.mnt_dir):
return True
self.assertIsNone(self.mount_process.poll())
retry(30, poll)
def __retry(url, maxcnt, interval, msg):
rc = self.p.post(url)
stat_url = CMD_STAT_URL_PTR % (self.url, rc['id'])
get_stat = lambda: self.p.get(stat_url)['success'] is True and self.p.get(stat_url)['active'] is False
retry(get_stat, maxcnt, interval, msg)
def __retry(url, maxcnt, interval, msg):
rc = self.p.post(url)
stat_url = CMD_STAT_URL_PTR % (self.url, rc['id'])
get_stat = lambda: self.p.get(stat_url)[
'success'] is True and self.p.get(stat_url)['active'] is False
retry(get_stat, maxcnt, interval, msg)
def get_jobs_from(self, icaos):
return common.retry(
self.get_query,
const.LINK +
'query=icao&search=jobsfrom&icaos={}'.format('-'.join(icaos)),
error_type=TooManyConnectionsException)