def component_tests_config():
config_file = os.getenv("COMPONENT_TESTS_CONFIG")
if config_file is None:
raise Exception(
"Need to provide path to config file in COMPONENT_TESTS_CONFIG")
with open(config_file, 'r') as stream:
config = yaml.safe_load(stream)
LOGGER.info(f"component tests base config {config}")
def _component_tests_config(additional_config={}, named_tunnel=True):
# Regression test for TUN-4177, running with proxy-dns should not prevent tunnels from running
additional_config["proxy-dns"] = True
additional_config["proxy-dns-port"] = 9053
if named_tunnel:
return NamedTunnelConfig(additional_config=additional_config,
cloudflared_binary=config['cloudflared_binary'],
tunnel=config['tunnel'],
credentials_file=config['credentials_file'],
ingress=config['ingress'])
return ClassicTunnelConfig(
additional_config=additional_config, cloudflared_binary=config['cloudflared_binary'],
hostname=config['classic_hostname'], origincert=config['origincert'])
return _component_tests_config
def create_tunnel(config, origincert_path, random_uuid):
# Delete any previous existing credentials file. If the agent keeps files around (that's the case in Windows) then
# cloudflared tunnel create will refuse to create the tunnel because it does not want to overwrite credentials
# files.
credentials_path = config["credentials_file"]
try:
os.remove(credentials_path)
except OSError:
pass
tunnel_name = "cfd_component_test-" + random_uuid
create_cmd = [
config["cloudflared_binary"], "tunnel", "--origincert",
origincert_path, "create", "--credentials-file", credentials_path,
tunnel_name
]
LOGGER.info(f"Creating tunnel with {create_cmd}")
subprocess.run(create_cmd, check=True)
list_cmd = [
config["cloudflared_binary"], "tunnel", "--origincert",
origincert_path, "list", "--name", tunnel_name, "--output", "json"
]
LOGGER.info(f"Listing tunnel with {list_cmd}")
cloudflared = subprocess.run(list_cmd, check=True, capture_output=True)
return json.loads(cloudflared.stdout)[0]["id"]
async def main():
global BROWSER
REQUEST_LIMITS['pushplus.plus'] = Limit(SleepTime(120))
async with contextlib.AsyncExitStack() as stack:
play = await stack.enter_async_context(async_playwright())
BROWSER = await play.chromium.launch()
stack.push_async_callback(play.stop)
stack.push_async_callback(BROWSER.close)
scheduler = AsyncIOScheduler(job_defaults={
'misfire_grace_time': 3600,
'coalesce': True
},
timezone=str(tzlocal.get_localzone()))
loop = asyncio.get_running_loop()
for s in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT):
loop.add_signal_handler(s, lambda: create_task(shutdown()))
event = asyncio.Event()
SHUTDOWN_VARIABLES['event'] = event
if RUN_IMMEDIATELY:
try:
task = create_task(run(return_cancelled_error=True))
TASKS.add(task)
await task
TASKS.discard(task)
LOGGER.warning('immediately running finished')
except CancelledError:
return
scheduler.add_job(run, 'cron', hour=7)
scheduler.add_job(run, 'cron', hour=12)
scheduler.add_job(run, 'cron', hour=19, minute=30)
scheduler.start()
stack.callback(scheduler.shutdown)
stack.push_async_callback(event.wait)
def render_GET(self, request):
LOGGER.info('Rendering TimelineView %s' % request.path)
request.setHeader('content-type', 'text/html')
processes = []
# This array will hold all of the data for the timeline view.
data = [['Timestamp']]
# Add the list of processes to the timeline table.
for row in self.db.get_process_cmdlines(name=self.process_name_filter):
processes.append(row[0])
data[0].append(row[1].strip())
LOGGER.debug('got process data: %s' % data)
# Now add the top-level PSS values for the processes to the table.
for row in self.db.get_process_stats(name=self.process_name_filter):
timestamp = row[0]
if timestamp != data[-1][0]:
# Moved onto a new snapshot
data.append([0] * (len(processes) + 1))
data[-1][0] = timestamp
# Add process for this snapshot
pos = 1 + processes.index(row[2])
data[-1][pos] = int(row[4])
flattenString(None,
TimelineElement('static/timeline.html',
data)).addCallback(self.renderOutput)
request.write(self.output)
return ""
async def func():
start = time.time()
while True:
if time.time() - start > timeout:
LOGGER.warning(
f'Login failed: {account_id} <<<{page.page_source=}>>>')
await notify('Login failed', f'{account_id}')
return False
# noinspection PyProtectedMember
try:
await page.goto(
'http://fresh.ahau.edu.cn/yxxt-v5/web/jkxxtb/tbJkxx.zf')
except playwright._impl._api_types.Error as e:
LOGGER.exception(e)
continue
await sleep(2)
await page.evaluate(js_codes.login())
await sleep(2)
await page.fill('#zh', account_id)
await page.fill('#mm', password)
await sleep(2)
await page.click('#dlan')
await sleep(2)
if page.url != 'http://fresh.ahau.edu.cn/yxxt-v5/web/jkxxtb/tbJkxx.zf':
await sleep(60 * 10)
continue
await sleep(5)
return True
def _parse_section(section_name, current_process, current_thread, maps, stats,
data):
if section_name == 'meminfo':
parse_meminfo(maps, data)
elif section_name == 'loadavg':
parse_loadavg(stats, data)
elif section_name == 'uptime':
parse_uptime(stats, data)
elif section_name == 'vmstat':
parse_vmstat(stats, data)
elif current_thread and section_name == 'stat':
parse_stat(current_thread, data)
elif current_process and section_name != '':
# Hit a new file, consolidate what we have so far.
if 'smaps' == section_name:
_save_smaps_region(current_process.maps, maps, current_process.pid,
data)
elif 'cmdline' == section_name:
# Some command lines have a number of empty arguments. Ignore
# that because it's not interesting here.
current_process.argv = filter(len, data.strip().split('\0'))
elif 'stat' == section_name:
parse_stat(current_process, data)
else:
LOGGER.error('Unrecognised section name: %s' % section_name)
async def ro(ctx: Context, dice: str, history: Dict[Union[str, Guild], np.ndarray]):
""" Roll a dice in NdN format openly """
LOGGER.debug("Rolling open")
result = await _r(ctx, dice, history)
if not result:
return
await ctx.send(result)
async def rp(ctx: Context, dice: str, history: Dict[Union[str, Guild], np.ndarray]):
""" Roll a dice in NdN format hidden, no info is shown """
LOGGER.debug("Rolled and passed")
result = await _r(ctx, dice, history)
if not result:
return
await ctx.send(f"{ctx.message.author.mention} rolled hidden and passed on!")
def post(self) -> HTMLResponse:
"""Make post request to page."""
LOGGER.debug(msg=f"POSTing to url: {self.url}")
response: HTMLResponse = \
self.session.post(url=self.url,
headers=self.session.headers,
data=self.payload)
return response
def __init__(self, bot: Bot, guild: Guild, choices: List[Tuple[int, str]]):
self.bot = bot
self.guild = guild
self.voters = set()
self.open = not any(choices)
self.choices = list(map(lambda choice: [*choice, 0], choices))
LOGGER.debug(self.choices)
async def rh(ctx: Context, dice: str, history: Dict[Union[str, Guild], np.ndarray]):
""" Roll a dice in NdN format hidden (PM) """
LOGGER.debug("Rolling hidden")
result = await _r(ctx, dice, history)
if not result:
return
await ctx.send(f"{ctx.message.author.mention} rolled hidden!")
await ctx.message.author.send(result)
def logout(self) -> HTMLResponse:
"""Login in to router admin"""
LOGGER.info('LOGGING OUT...')
response: HTMLResponse = self.get()
assert response.html.search('You are {} logged out.'), (
f'Failed to logout. HTML: {response.html.html}')
LOGGER.info('LOGGED OUT...')
return response
def delete_tunnel(config):
credentials_path = config["credentials_file"]
delete_cmd = [
config["cloudflared_binary"], "tunnel", "--origincert",
config["origincert"], "delete", "--credentials-file", credentials_path,
"-f", config["tunnel"]
]
LOGGER.info(f"Deleting tunnel with {delete_cmd}")
subprocess.run(delete_cmd, check=True)
async def shutdown():
if SHUTDOWN_VARIABLES['killed']:
os._exit(1)
SHUTDOWN_VARIABLES['killed'] = True
LOGGER.warning('Exiting')
for t in TASKS:
t.cancel()
if TASKS:
await asyncio.wait(TASKS)
LOGGER.debug(f'{TASKS=}')
SHUTDOWN_VARIABLES['event'].set()
def list_devices(self):
"""List devices & statuses of the internal DHCP server for the LAN"""
LOGGER.info('Getting list of devices on network.')
header_row: Element = self.html.find(self.TABLE_HEADER, first=True)
rows: List[Element] = self.html.find(self.TABLE_ROWS)
table_header_map: dict = {
str(i): x.text
for i, x in enumerate(header_row.find('td'))
}
print(' '.join(table_header_map.values()))
for row in rows:
print(' '.join([e.text for e in row.find('td')]))
async def main():
""" Main function """
env = Env()
env.read_env()
setup_logger(env.int("LOG_LEVEL", logging.INFO),
env.path("LOG_FILE", None))
async with aiohttp.ClientSession() as session:
bot = create_bot(env, session)
LOGGER.debug("Starting bot")
await bot.start(env.str("BOT_TOKEN"))
def enable_wifi(self):
"""Disable WiFi"""
LOGGER.info('DISABLING WIFI...')
wireless_url = f'{self.url}/wlanRadio.asp'
wireless_form = f'{self.url}/goform/wlanRadio'
response: HTMLResponse = self.session.get(wireless_url)
html = response.html
current_params = get_page_selected_selects_as_dict(html=html)
current_params.update({'WirelessEnable': '1'})
response: HTMLResponse = self.session.post(
url=wireless_form, data=current_params
)
next_page = response.html
current_params = get_page_selected_selects_as_dict(html=next_page)
assert current_params['WirelessEnable'] == '1'
def parse(self, data, out):
out['meminfo'] = MemoryStats()
for line in data.split('\n'):
parts = re.split('[ :]+', line.strip())
if len(parts) < 2:
LOGGER.debug('Skipping meminfo line that is too short: %s' %
line)
elif len(parts) == 2:
# This is a number. eg HugePages_Total, HugePages_Free,
# HugePages_Rsvd, HugePages_Surp
out['meminfo'].meminfo[parts[0]] = int(parts[1])
else:
# These are sizes, with unit kB in the third column.
# eg. AnonHugePages: 2355200 kB
out['meminfo'].meminfo[parts[0]] = int(parts[1]) * 1024
return out
def parse_meminfo(stats, data):
if not isinstance(stats, MemoryStats):
raise TypeError('%s is not of type MemoryStats' % type(stats))
for line in data.split('\n'):
parts = re.split('[ :]+', line.strip())
if len(parts) < 2:
LOGGER.debug('Skipping meminfo line that is too short: %s' % line)
elif len(parts) == 2:
# This is a number. eg HugePages_Total, HugePages_Free,
# HugePages_Rsvd, HugePages_Surp
stats.meminfo[parts[0]] = int(parts[1])
else:
# These are sizes, with unit kB in the third column.
# eg. AnonHugePages: 2355200 kB
stats.meminfo[parts[0]] = int(parts[1]) * 1024
def login(self) -> HTMLResponse:
"""Login in to router admin"""
LOGGER.info('LOGGING IN...')
response: HTMLResponse = self.post()
rows: List[Element] = response.html.find(self.TABLE_ROWS)
try:
assert rows
except AssertionError:
error_element = response.html.find('font[style="color: red"]',
first=True)
if error_element.text == self.USER_OR_PASSWORD_ERROR:
raise PermissionError(self.USER_OR_PASSWORD_ERROR)
else:
raise Exception(
f"Table rows but none found. HTML: \n{response.html.html}")
LOGGER.info('LOGGED IN...')
return response
async def _r(ctx: Context, dice: str, history: Dict[Union[str, Guild], np.ndarray]):
""" String parsing and rolling for all dice rolling functions """
try:
rolls, limit = map(int, dice.split("d"))
except Exception:
await ctx.send("Format has to be in NdN!")
return None
if rolls > 200 or limit > 200:
await ctx.send("Pls use smaller numbers '_'")
return None
result = np.random.randint(low=1, high=limit + 1, size=rolls)
result = np.sort(result)[::-1]
LOGGER.debug(result)
guild = ctx.guild if ctx.guild else "default"
LOGGER.debug(guild)
history[guild] = result
result = ", ".join(map(str, result))
return result
def component_tests_config():
config_file = os.getenv("COMPONENT_TESTS_CONFIG")
if config_file is None:
raise Exception(
"Need to provide path to config file in COMPONENT_TESTS_CONFIG")
with open(config_file, 'r') as stream:
config = yaml.safe_load(stream)
LOGGER.info(f"component tests base config {config}")
def _component_tests_config(additional_config={},
cfd_mode=CfdModes.NAMED,
run_proxy_dns=True):
if run_proxy_dns:
# Regression test for TUN-4177, running with proxy-dns should not prevent tunnels from running.
# So we run all tests with it.
additional_config["proxy-dns"] = True
additional_config["proxy-dns-port"] = PROXY_DNS_PORT
else:
additional_config.pop("proxy-dns", None)
additional_config.pop("proxy-dns-port", None)
if cfd_mode is CfdModes.NAMED:
return NamedTunnelConfig(
additional_config=additional_config,
cloudflared_binary=config['cloudflared_binary'],
tunnel=config['tunnel'],
credentials_file=config['credentials_file'],
ingress=config['ingress'])
elif cfd_mode is CfdModes.CLASSIC:
return ClassicTunnelConfig(
additional_config=additional_config,
cloudflared_binary=config['cloudflared_binary'],
hostname=config['classic_hostname'],
origincert=config['origincert'])
elif cfd_mode is CfdModes.PROXY_DNS:
return ProxyDnsConfig(
cloudflared_binary=config['cloudflared_binary'])
else:
raise Exception(f"Unknown cloudflared mode {cfd_mode}")
return _component_tests_config
def expect_address_connections(self, tmp_path, component_tests_config, protocol, edge_ip_version, assert_address_type):
config = component_tests_config(
self._extra_config(protocol, edge_ip_version))
config_path = write_config(tmp_path, config.full_config)
LOGGER.debug(config)
with CloudflaredCli(config, config_path, LOGGER):
wait_tunnel_ready(tunnel_url=config.get_url(),
require_min_connections=4)
cfd_cli = CloudflaredCli(config, config_path, LOGGER)
tunnel_id = config.get_tunnel_id()
info = cfd_cli.get_tunnel_info(tunnel_id)
connector_id = get_tunnel_connector_id()
connector = next(
(c for c in info["conns"] if c["id"] == connector_id), None)
assert connector, f"Expected connection info from get tunnel info for the connected instance: {info}"
conns = connector["conns"]
assert conns == None or len(
conns) == 4, f"There should be 4 connections registered: {conns}"
for conn in conns:
origin_ip = conn["origin_ip"]
assert origin_ip, f"No available origin_ip for this connection: {conn}"
assert_address_type(origin_ip)
def create_bot(env: Env, session: aiohttp.ClientSession) -> Bot:
""" Setup the Bot """
intents = Intents.default()
intents.members = True
bot = Rollo(("!", "?", "->"), intents=intents)
async def on_ready():
LOGGER.info("Logged in as %s: %d", bot.user.name, bot.user.id)
async def on_command_error(_, error):
LOGGER.warning("Command error: %s", error)
bot.add_listener(on_ready, "on_ready")
bot.add_listener(on_command_error, "on_command_error")
bot.add_cog(General(bot))
bot.add_cog(Meiern())
if env.str("TENOR_TOKEN", None) is not None:
LOGGER.info("Tenor API Key found. Enabling GIF posting!")
bot.add_cog(Memes(session, env.str("TENOR_TOKEN")))
return bot
def _parse_section(section_name, current_process, current_thread, data, out):
try:
parser = parsers.get_parser(section_name)
parser.parse(data, out)
except:
pass
if current_thread and section_name == 'stat':
_save_stat(current_thread, out['stat'])
elif current_process and section_name != '':
# Hit a new file, consolidate what we have so far.
if 'smaps' == section_name:
_save_smaps_region(current_process.maps, out['meminfo'],
current_process.pid, data)
elif 'cmdline' == section_name:
# Some command lines have a number of empty arguments. Ignore
# that because it's not interesting here.
current_process.argv = filter(len, data.strip().split('\0'))
elif 'stat' == section_name:
_save_stat(current_process, out['stat'])
else:
LOGGER.error('Unrecognised section name: %s' % section_name)
async def vote(self, ctx: Context, *choices: str, time=20):
""" Create a vote. (Alt command: v) """
# Check if voting is possible
guild = ctx.guild
if ctx.guild is None:
await ctx.send("This feature is only supported in guilds")
return
vote = self.votes.get(guild)
if vote:
await ctx.send("There is already a vote running")
return
# Attach a number to each choice
choice_enum: List[Tuple[int, str]] = list(
enumerate(
map(lambda choice: choice.strip(" ,\n").lower(), choices), 1))
vote = Vote(self.bot, guild, choice_enum)
self.votes[guild] = vote
LOGGER.debug("Started voting listener")
self.bot.add_listener(vote.on_vote, name="on_message")
choice_text = ("\n".join(
map(lambda choice: f"{choice[0]}\t\u21D2 \t{choice[1]}",
choice_enum)) if choice_enum else "Open voting")
await ctx.send(f"Voting started for {time}s.\n{choice_text}")
await asyncio.sleep(time)
self.bot.remove_listener(vote.on_vote, name="on_message")
LOGGER.debug("Removed voting listener")
results = vote.format_results()
hist = vote.termogram()
if hist is not None:
await ctx.send(f"Voting finished!\n{hist}")
else:
await ctx.send(f"Voting finished!\n{results}")
del self.votes[guild]
def parse_smaps_header(header):
info = MemoryRegion(free=False)
# Example line is:
# 011e6000-01239000 rw-p 00000000 00:00 0 [heap]
# 8ec00000-8ec01000 rw-s 00000000 00:14 20 /dev/shm/NS2371 (deleted)
# All numbers are hex except for the inode
parts = header.split()
LOGGER.debug('Parsing smaps header %s' % header)
# Parse the address range
info.start_addr, info.end_addr = [long(x, 16) for x in parts[0].split('-')]
# Parse the permissions
permissions = parts[1]
info.permissions.readable = "r" in permissions
info.permissions.writable = "w" in permissions
info.permissions.executable = "x" in permissions
info.permissions.private = "p" in permissions
info.permissions.shared = "s" in permissions
info.offset = long(parts[2], 16)
# eg. 08:06
info.major_dev, info.minor_dev = [int(x, 16) for x in parts[3].split(':')]
# The inode isn't a hex number
info.inode = int(parts[4])
# eg. [heap]
# or /dev/shm/NS2371
if len(parts) > 5:
info.name = parts[5]
info.deleted = header.endswith('(deleted)')
return info
async def func():
async with contextlib.AsyncExitStack() as stack:
if not RUN_IMMEDIATELY:
await sleep(random() * 30 * 60)
account_id = account['account-id']
password = account['password']
context, page = await new_context()
stack.push_async_callback(context.close)
stack.push_async_callback(page.close)
logged = await login(page, account_id, password)
if not logged:
return
html = HTML(await page.content())
source = await get_script_source(html=html)
if SCRIPT_SOURCE:
if diff := '\n'.join(
get_diff(SCRIPT_SOURCE, source.splitlines())):
await handle_page_changing(diff, source)
return
name = html.xpath("//input[@id='xm']/@value")[0]
await page.evaluate(js_codes.submit())
await sleep(2)
async with page.expect_response('**/tbBcJkxx.zf') as response:
await page.click("//button[text()='提交']")
await sleep(5)
response = await (await response.value).json()
if response['status'] == 'success':
await page.wait_for_selector("//div[text()='保存数据成功']",
state='attached')
LOGGER.warning(f'Success: {account_id} {name}')
else:
LOGGER.warning(
f'Submit failed: {account_id} {name} <<<{source=}>>>')
await notify('Submit failed', f'{account_id} {name}')
def parse_smaps_memory_region(pid, lines, has_header=True):
"""Parse a whole smaps region, which may look like:
7f5c8550e000-7f5c85554000 r--p 00000000 08:06 1309629 /fonts/Arial_Bold.ttf
Size: 280 kB
Rss: 152 kB
Pss: 86 kB
Shared_Clean: 132 kB
Shared_Dirty: 12 kB
Private_Clean: 20 kB
Private_Dirty: 1 kB
Referenced: 152 kB
Anonymous: 2 kB
AnonHugePages: 3 kB
Shared_Hugetlb: 4 kB
Private_Hugetlb: 5 kB
Swap: 6 kB
SwapPss: 7 kB
KernelPageSize: 8 kB
MMUPageSize: 9 kB
Locked: 10 kB
VmFlags: rd mr mw me sd"""
has_header = is_memory_region_header(lines[0])
if has_header:
region = parse_smaps_header(lines[0])
if region.name == '[vsyscall]':
return None
lines = lines[1:]
else:
region = MemoryRegion(free=False)
region.pid = pid
global _smaps_string_mappings
for line in lines:
LOGGER.debug('Parsing line: %s' % line)
parts = re.split('[ :]+', line.strip())
if len(parts) < 2:
LOGGER.debug('Skipping smaps line that is too short: %s' % line)
elif 'Size' == parts[0]:
# We calculate the size from the address ranges instead.
pass
elif 'VmFlags' == parts[0]:
region.vm_flags = parts[1:]
else:
# All other lines should be an amount of some type of memory.
try:
region.__dict__[_smaps_string_mappings[parts[0]]] = int(
parts[1]) * 1024
except KeyError:
LOGGER.warn("Line not recognised: '%s'" % line)
return region
def main(args):
import logging
if args.verbose:
LOGGER.setLevel(logging.DEBUG)
else:
LOGGER.setLevel(logging.INFO)
# Get the database handle
db = Database(args.db, args.overwrite)
for i in range(args.count):
if i > 0:
time.sleep(args.period)
# Read all the data we need
system_stats, processes, memory_stats = read_stats(args)
LOGGER.info('Found {} process(es) and {} used memory fragments'.format(
len(processes), len(memory_stats)))
LOGGER.info('Regions: %s' % memory_stats)
db.add(args.host if len(args.host) else '[local]', system_stats, memory_stats, processes)
