def setup():
if utils.get_host().startswith("maya"):
add_filemenu_maya()
if utils.get_host().startswith("nuke"):
add_filemenu_nuke()
print("tapp: Loaded successfully.")
def get_docked_widget():
if "maya" in utils.get_host():
return get_maya_dock_widget()
if "nuke" in utils.get_host():
return get_nuke_dock_widget()
return None
def set_neighbors(self):
self.neighbors = utils.find_neighbors(utils.get_host(), self.port,
NEIGHBORS_IP_RANGE_NUM[0],
NEIGHBORS_IP_RANGE_NUM[1],
BLOCKCHAIN_PORT_RAMGE[0],
BLOCKCHAIN_PORT_RAMGE[1])
logger.info({'action': 'set_neighbors', 'neighbors': self.neighbors})
def reply_eulam_forum(post_url, src):
logger = utils.RAPLogger(post_url)
host = utils.get_host(post_url)
s = utils.RAPSession(src)
# Step 1: 登录
if not login_eulam_forum(s, src):
logger.error(' Login Error')
return (False, str(logger))
logger.info(' Login OK')
r = s.get(post_url)
soup = BeautifulSoup(r.content)
form = soup.find('form', attrs={'name': 'form'})
payload = utils.get_datadic(form, CHARSET)
payload['Body'] = src['content'].decode('utf8').encode(CHARSET)
payload['BBSXPCodeForm'] = ''
r = s.post(host + 'ReTopic.asp',
data=payload,
headers={'Referer': post_url})
if u'操作成功'.encode(CHARSET) not in r.content:
logger.error('Reply Error')
return (False, str(logger))
logger.info('Reply OK')
return (True, str(logger))
def get_summary(self):
res = []
path_dir = self.mars_config.get_base_path()
snap_times_list = os.listdir(path_dir)
snap_times_list.sort(reverse=True)
for snap_time in snap_times_list:
try:
time_stamp = int(snap_time)
time_str = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time_stamp))
cur_summary = {
'time':
time_str,
GROUPS_NAME:
len(get_group(self.mars_config, snap_time)),
DEVICE_NAME:
len(get_devices(self.mars_config, snap_time)),
DEVICE_CONFIG_NAME:
len(get_devices_configs(self.mars_config, snap_time)),
HOSTS_NAME:
len(get_host(self.mars_config, snap_time)),
LINKS_NAME:
len(get_link(self.mars_config, snap_time)),
FLOW_NAME:
len(get_flow(self.mars_config, snap_time)),
}
res.append(cur_summary)
except ValueError as e:
pass
return res
def host_up(ctx, force, restart, recreate, context=None):
if context is None:
context = ctx
host_name = context.parent.parent.params.get('host')
host = utils.get_host(host_name)
host.config = utils.get_host_config(host_name)
host.up()
def google_page(path='zapretno.info/klip-haski-iuda'):
logger.info(f'Fetching {path}')
if 'localhost' in request.host_url:
if not path.startswith('http'):
path = 'http://' + path
domain, suffix = get_host(path)
domain = f'http://{domain}'
else:
domain, suffix = request.host_url, path
resp = requests.get(f'{domain}{suffix}',
stream=True,
params=request.args,
timeout=5)
all_urls = find_all_relative_urls(
resp.content.decode('utf-8', errors='ignore'))
logger.info('Urls in the web page: ' + str(all_urls))
if all_urls:
new_content = replace_with_host(
urls=all_urls,
host=domain,
content=resp.content.decode(encoding='utf-8'))
logger.info('Get new content')
resp._content = new_content.encode('utf-8')
logger.info(f'Got {resp.status_code} response from {path}')
return Response(generate(response=resp), )
def set_neighbours(self):
"""
条件に沿ったnodeを検索する．
See Also
----------
NEIGHBOURS_IP_RANGE_NUM : tuple
NEIGHBOURS_IP_RANGE_NUM[0] : int
start_ip_range
NEIGHBOURS_IP_RANGE_NUM[1] : int
end_ip_range
BLOCKCHAIN_PORT_RANGE : tuple
BLOCKCHAIN_PORT_RANGE[0] : int
start_port
BLOCKCHAIN_PORT_RANGE[1] : int
end_port
"""
self.neighbours = utils.find_neighbours(utils.get_host(), self.port,
NEIGHBOURS_IP_RANGE_NUM[0],
NEIGHBOURS_IP_RANGE_NUM[1],
BLOCKCHAIN_PORT_RANGE[0],
BLOCKCHAIN_PORT_RANGE[1])
logger.info({
"action": "set_neighbours",
"neighbours": self.neighbours
})
def send_ip_db(ip_src, session, user):
global count
hostname = "unknown host"
try:
hostname = socket.gethostbyaddr(ip_src)
except socket.herror:
hostname = "unknown"
# print "{} : packet from ip={} : hostname={}".format(count,str(ip_src), hostname[0])
subnet = "{}.0/24".format(ip_src.rsplit(".",1)[0])
new_ip = IPAddress(IP=ip_src)
new_subnet = Subnet(subnet=subnet)
try:
session.add(new_ip)
session.commit()
print "adding new ip {} : {}".format(ip_src, get_host(ip_src))
except IntegrityError:
# print "could not add new_ip because of integrity error - {}".format(new_ip.IP)
session.rollback()
new_ip = session.query(IPAddress).filter(IPAddress.IP==ip_src)[0]
try:
session.add(new_subnet)
session.commit()
#print "subnet = {}".format(subnet.replace("/24",""))
#print "adding new subnet {} : {}".format(subnet, get_host(subnet.replace("/24","")))
except IntegrityError:
session.rollback()
new_subnet = session.query(Subnet).filter(Subnet.subnet==subnet)[0]
new_visit = VisitByUser(user=user,IP=new_ip, subnet = new_subnet, dateTime=datetime.datetime.now())
session.add(new_visit)
session.commit()
count += 1
def goto_login():
next = request.args.get('next', '')
if get_host(next) != request.host:
return redirect('/')
params = urllib.urlencode({'next':next.encode('utf-8')})
url = '%s?%s' % (ACCOUNTS_LOGIN, params)
return redirect(url)
def remove(ctx):
workspace = Workspace(utils.get_host(ctx.parent.params.get('host')))
try:
workspace.remove()
return
except subprocess.CalledProcessError as e:
click.echo('Unable to remove the workspace.', err=True)
exit(1)
def host_docker_env(ctx, host_name):
if host_name is None:
host_name = ctx.parent.parent.params.get('host')
host = utils.get_host(host_name)
host.config = utils.get_host_config(host_name)
try:
host.ping()
click.echo('Running')
except base_host.HostDownException:
click.echo("Stopped'")
def state(ctx):
workspace = Workspace(utils.get_host(ctx.parent.params.get('host')))
state = None
try:
state = workspace.state
except base_host.HostDownException:
state = 'host-down'
except subprocess.CalledProcessError as e:
click.echo('Unable to get workspace state.', err=True)
exit(1)
click.echo(state)
def backend_connect(self, timestamp, request_id, replica_set_id, request_url):
"""
Process BackendConnect event
Extract host here for optimisation puropose
"""
host = utils.get_host(request_url)
self.open_requests[request_id].backend_connect(replica_set_id, host)
if replica_set_id not in self.backend_stats:
self.backend_stats[replica_set_id] = ReplicaSetStat(replica_set_id)
self.backend_stats[replica_set_id].backend_connect(host)
def thumb_up_163(post_url, src):
logger = utils.RAPLogger(post_url)
sess = utils.RAPSession(src)
resp = sess.get(post_url)
resp = sess.post(src['extra']['target_url'],
headers={
'Referer': post_url,
'Host': utils.get_host(post_url),
'X-Requested-With': 'XMLHttpRequest',
})
logger.info(resp.content)
return (True, str(logger))
def ssh_command(ctx, command, force, cmd=None, context=None):
if context is None:
context = ctx
host_name = context.parent.parent.params.get('host')
host = utils.get_host(host_name)
host.config = utils.get_host_config(host_name)
try:
click.echo(' '.join(host.ssh_command(command=cmd)))
return
except base_host.HostDownException:
if not utils.confirm_host_up(force=force, host=host):
return
ctx.invoke(ssh, command=command, force=force, cmd=cmd, context=context)
def ssh_config(ctx, force, context=None):
if context is None:
context = ctx
host_name = context.parent.parent.params.get('host')
host = utils.get_host(host_name)
host.config = utils.get_host_config(host_name)
try:
click.echo(host.flat_ssh_config)
return
except base_host.HostDownException:
if not utils.confirm_host_up(force=force, host=host):
return
ctx.invoke(ssh_config, force=force, context=context)
def __init__(self, my_port=50082, c_host=None, c_port=None, callback=None):
self.client_state = STATE_INIT
print('Initializing ClientCore...')
self.my_ip = utils.get_host()
print('Server IP address is set to ... ', self.my_ip)
self.my_port = my_port
self.my_core_host = c_host
self.my_core_port = c_port
self.blockchain = BlockChain()
self.wallet = Wallet()
self.cm = ConnectionManager4Edge(self.my_ip, self.my_port, c_host,
c_port, self.__handle_message)
self.callback = callback
def build(ctx, no_cache, force, context=None):
if context is None:
context = ctx
host = utils.get_host(context.parent.params.get('host'))
workspace = Workspace(host)
try:
workspace.build(no_cache=no_cache)
return
except base_host.HostDownException:
if not utils.confirm_host_up(force=force, host=host):
return
except WorkspaceDownException:
workspace.up()
ctx.invoke(build, no_cache=no_cache, force=force, context=context)
def reply_backchina_forum(post_url, src):
"""倍可亲回复模块
@param sess: requests.Session()
@type sess: Session
@param post_url: 帖子地址
@type post_url: str
@param src: 用户名，密码，回复内容，等等。
@type src: dict
@return: 是否登录成功
@rtype: bool
"""
# Returnable logger
logger = utils.RAPLogger(post_url)
host = utils.get_host(post_url)
sess = utils.RAPSession(src)
# Step 1: 登录
if not login_backchina(sess, src):
logger.error(' Login Error')
return (False, str(logger))
logger.info(' Login OK')
# Step 2: Load post page
# 获取所需发帖页面，查找与{'id':'fastpostform'}匹配的form标签
resp = sess.get(post_url)
soup = BeautifulSoup(resp.content)
form = soup.find('form', attrs={'id': 'fastpostform'})
# Step 3: Submit
# 回复内容
payload = utils.get_datadic(form)
if 'subject' in src:
payload['subject'] = src['subject']
payload['message'] = src['content']
#发送post包
resp = sess.post(host + form['action'], data=payload)
#判断回帖后页面是否含有回帖内容，若存在则证明回帖成功，否则失败
if src['content'] in resp.content:
logger.info('Reply OK')
else:
logger.error('Reply Error: Reply Error, please try again !')
return (False, str(logger))
return (True, str(logger))
def host_docker_env(ctx, shell, host_name):
if host_name is None:
host_name = ctx.parent.parent.params.get('host')
host = utils.get_host(host_name)
host.config = utils.get_host_config(host_name)
try:
host.ping()
output = [
'export DOCKER_HOST="tcp://{}:2375"'.format(host.ip),
'export DOCKER_MACHINE_NAME="{}"'.format(host_name),
]
click.echo('\n'.join(output))
except base_host.HostDownException:
click.echo("echo 'Host is down!'")
def post_wailaike_forum(post_url, src):
""" 外来客论坛发主贴函数
@param post_url: 板块地址 如：http://www.wailaike.net/group_post?gid=1
@type post_url: str
@param src: 用户名，密码，标题，主帖内容，等等。用户名：[email protected] 密码：wenshen4921119
@type src: dict
@return: 是否发帖成功，帖子URL
@rtype: bool,str
"""
# Returnable logger
logger = utils.RAPLogger(post_url)
host = utils.get_host(post_url)
sess = utils.RAPSession(src)
# Step 1: 登录
if not login_wailaike(sess, src):
logger.error(' Login Error')
return ('', str(logger))
logger.info(' Login OK')
gid = re.findall(r'gid=(\d*)', post_url)[0]
resp = sess.get('http://www.wailaike.net/newpost?gid='+gid)
soup = BeautifulSoup(resp.content)
# 获得发帖form
form = soup.find('form', attrs={'id': 'editor'})
# 构造回复参数
payload = utils.get_datadic(form)
payload['title'] = src['subject'].decode('utf8').encode(CHARSET)
payload['rstbody'] = src['content'].decode('utf8').encode(CHARSET)
resp = sess.get('http://www.wailaike.net/time.php')
payload['time'] = re.findall('"time":"(.*?)"', resp.content)[0]
# 发送发帖post包
resp = sess.post('http://www.wailaike.net/newpost?gid='+gid, data=payload)
# By sniper 2015-2-1
# 标题中的'('和')'等需要在正则表达式中转义
# 如：[转帖]ZT) 汉服是FQ闹的大笑话
subject = re.escape(src['subject'])
url = re.findall(r'<h3 class="titles-txt"><a href=\"(.*?)\" target="_blank">' + subject + '</a></h3>',resp.content)[0]
url = "http://www.wailaike.net" + url
# 如果url未成功匹配，则抛出异常，Post Error
logger.info(' Post OK')
return (url, str(logger))
def reply_canyu(post_url, src):
"""参与网回帖模块
@author: sky
@since: 2015-01-05
@param sess: requests.Session()
@type sess: Session
@param post_url: 帖子地址
@type post_url: str
@param src: 用户名，密码，回复内容，等等。
@type src: dict
@return: 是否登录成功
@rtype: bool
"""
logger = RAPLogger(post_url)
host = get_host(post_url)
sess = RAPSession(src)
# Step 1: 获取回帖页面
resp = sess.get(post_url)
soup = BeautifulSoup(resp.content)
form = soup.find('form', attrs={'id': 'remarkForm'})
# Step 2: 提交回帖
# 回复内容
payload = {}
payload = get_datadic(form)
payload['face'] = '1'
payload['body'] = src['content'].decode('utf8').encode(CHARSET)
if 'nickname' in src:
payload['username'] = src['nickname'].decode('utf8').encode(CHARSET)
else:
payload['username'] = u'匿名'.encode(CHARSET)
#发送post包
resp = sess.post(host + form['action'], data=payload)
#再次请求原网页，查看是否已经有回帖内容
resp = sess.get(post_url)
#判断回帖后页面是否含有回帖内容，若存在则证明回帖成功，否则失败
if src['content'].decode('utf8').encode(CHARSET) in resp.content:
logger.info('Reply OK')
else:
logger.error('Reply Error, please try again !')
return (False, str(logger))
return (True, str(logger))
def post_powerapple_forum(post_url, src):
""" 超级苹果论坛发主贴函数
@param post_url: 板块地址 如：http://bbs.powerapple.com/forum.php?mod=forumdisplay&fid=50
@type post_url: str
@param src: 用户名，密码，标题，主帖内容，等等。
@type src: dict
@return: 是否发帖成功，帖子URL
@rtype: bool,str
"""
logger = utils.RAPLogger(post_url)
host = utils.get_host(post_url)
sess = utils.RAPSession(src)
# Step 1: 登录
if not login_powerapple(sess, src):
logger.error(' Login Error')
return ('', str(logger))
logger.info(' Login OK')
# Step 2: Load post page
fid = re.findall(r'fid=(\d+)', post_url)[0]
# 获取所需发帖页面，查找与{'id':'fastpostform'}匹配的form标签
resp = sess.get(
'http://bbs.powerapple.com/forum.php?mod=post&action=newthread&fid=' +
fid)
soup = BeautifulSoup(resp.content)
form = soup.find('form', attrs={'id': 'postform'})
# Step 3: Submit
# 回复内容
payload = get_datadic(form)
payload['subject'] = src['subject']
payload['message'] = src['content']
payload['typeid'] = '138'
#发送post包
resp = sess.post(host + form['action'], data=payload)
#获取回帖页面content的HTML
print_to_file(resp.content)
#判断回帖后页面是否含有回帖内容，若存在则证明回帖成功，否则失败
if src['subject'] in resp.content:
logger.info('Post OK')
else:
logger.error('Reply Error: please try again !')
return ('', str(logger))
return (resp.url, str(logger))
def init_snap_data(self, snap_time_str):
print('Start to trace the ' +
UseStyle(snap_time_str, fore='green', mode='underline') +
' snap data\n')
snap_time = format_time_string_2_number(snap_time_str)
self.snap_time = snap_time
self.device_config_obj = DeviceConfigs.initialize_with(
self.mars_config, get_devices_configs(self.mars_config, snap_time))
self.hosts_obj = Hosts.initialize_with(
self.mars_config, get_host(self.mars_config, snap_time))
self.group_obj = Groups.initialize_with(
self.mars_config, get_group(self.mars_config, snap_time))
self.flow_obj = Flows.initialize_with(
self.mars_config, get_flow(self.mars_config, snap_time))
self.link_obj = Links.initialize_with(
self.mars_config, get_link(self.mars_config, snap_time))
def ssh_command(ctx, command, force, recreate, cmd=None, context=None):
if context is None:
context = ctx
host = utils.get_host(context.parent.params.get('host'))
workspace = Workspace(host)
try:
if recreate:
workspace.recreate()
click.echo(' '.join(workspace.ssh_command(cmd)))
return
except base_host.HostDownException:
if not utils.confirm_host_up(force=force, host=host):
return
except WorkspaceDownException:
workspace.up()
ctx.invoke(ssh, command=command, force=force, recreate=recreate, cmd=cmd, context=context)
def __init__(self,
my_port=50082,
core_node_host=None,
core_node_port=None):
self.server_state = STATE_INIT
print('Initializing server...')
self.my_ip = utils.get_host()
print('Server IP address is set to ... ', self.my_ip)
self.my_port = my_port
self.cm = ConnectionManager(self.my_ip, self.my_port,
self.__handle_message)
self.core_node_host = core_node_host
self.core_node_port = core_node_port
self.miners_wallet = Wallet()
self.blockchain = BlockChain(self.miners_wallet.blockchain_address)
self.mining_semaphore = threading.Semaphore(1)
self.__print_info()
def host_ip(ctx, force, restart, recreate, context=None):
if context is None:
context = ctx
host_name = context.parent.parent.params.get('host')
host = utils.get_host(host_name)
host.config = utils.get_host_config(host_name)
try:
ip = host.ip
if recreate:
recreate = force or click.confirm("Are you sure you want to recreate '{}'?".format(host.name))
if not recreate:
return
host.recreate()
elif restart:
host.restart()
click.echo(ip)
except base_host.HostDownException:
if not utils.confirm_host_up(force=force, host=host):
return
ctx.invoke(host_ip, force=True, restart=restart, recreate=recreate, context=context)
def post_1dpw_forum(post_url, src):
""" 加国华人网发主贴函数
@param post_url: 板块地址 如：http://bbs.1dpw.com/forum-71-1.html
@type post_url: str
@param src: 用户名，密码，标题，主帖内容，等等。
@type src: dict
@return: 是否发帖成功，帖子URL
@rtype: bool,str
"""
host = utils.get_host(post_url)
logger = utils.RAPLogger(post_url)
sess = utils.RAPSession(src)
# Step 1: 登录
if not login_1dpw(post_url, sess, src):
logger.error(' Login Error')
return ('', str(logger))
logger.info(' Login OK')
resp = sess.get(post_url)
soup = BeautifulSoup(resp.content)
# 获得回复form
form = soup.find('form', attrs={'id': 'fastpostform'})
# 构造回复参数
payload = utils.get_datadic(form)
payload['subject'] = src['subject']
payload['message'] = src['content']
payload['posttime'] = int(time.time())
# 发送登录post包
resp = sess.post(host + form['action'] + '&inajax=1', data=payload)
# 若指定字样出现在response中，表示发帖成功
if '主题已发布' not in resp.content:
logger.error(' Post Error')
return ('', str(logger))
logger.info(' Post OK')
url = host + re.findall(r'succeedhandle_fastnewpost\(\'(.*?)\'', resp.content)[0]
print url
return (url, str(logger))
def reply_creaders_news(post_url, src):
logger = utils.RAPLogger(post_url)
host = utils.get_host(post_url)
s = utils.RAPSession(src)
r = s.get(post_url)
payload = {
'news_id': re.findall('news_id=(\d+)', r.content)[0],
'r_nid': re.findall('r_nid=(\d+)', r.content)[0],
'username': src['username'],
'password': src['password'],
'replyid': 0,
# The charset of this page is `gb2312` absolutely, but it seems that
# `saytext` receives `utf8` only.
'saytext': src['content'],
}
r = s.post(host + '/headline/postcomment.php', data=payload)
if u'评论成功'.encode('gb2312') not in r.content:
logger.error('Reply Error')
return (False, str(logger))
logger.info('Reply OK')
return (True, str(logger))
def host_ssh(ctx, command, force, restart, recreate, cmd=None, context=None):
if context is None:
context = ctx
host_name = context.parent.parent.params.get('host')
host = utils.get_host(host_name)
host.config = utils.get_host_config(host_name)
try:
host.ping()
if recreate:
recreate = force or click.confirm("Are you sure you want to recreate '{}'?".format(host.name))
if not recreate:
return
host.recreate()
elif restart:
host.restart()
result = host.ssh(command=cmd, stdout=True)
if result is not None:
click.echo(''.join(result))
except base_host.HostDownException:
if not utils.confirm_host_up(force=force, host=host):
return
ctx.invoke(host_ssh, command=command, force=True, restart=restart, recreate=recreate, cmd=cmd, context=context)
def ssh(ctx, force, recreate, rebuild, command=False, cmd=None, context=None):
if context is None:
context = ctx
host = utils.get_host(context.parent.params.get('host'))
workspace = Workspace(host)
try:
if rebuild:
workspace.build()
recreate = True
if recreate:
workspace.recreate()
sleep(1)
workspace.ssh(command=cmd)
return
except base_host.HostDownException:
if not utils.confirm_host_up(force=force, host=host):
return
except WorkspaceDownException:
workspace.up()
except ssh_utils.SshException as e:
exit(e.returncode)
ctx.invoke(ssh, force=force, recreate=recreate, rebuild=rebuild, command=command, cmd=cmd, context=context)
def show_snap_hosts(self, snap_time_str):
print_normal('Show the ' +
UseStyle(snap_time_str, fore='green', mode='underline') +
' hosts.')
snap_time = format_time_string_2_number(snap_time_str)
host_obj = Hosts.initialize_with(self.mars_config,
get_host(self.mars_config, snap_time))
devices_config_obj = DeviceConfigs.initialize_with(
self.mars_config, get_devices_configs(self.mars_config, snap_time))
for device_config in devices_config_obj.get_data():
print_normal_start('Device Name : ' + device_config['name'],
color='yellow')
for host in host_obj.get_data():
for location in host['locations']:
if location['elementId'] == device_config['id']:
print_normal_center(
host_to_line_string(host, devices_config_obj))
print_normal_end('')
print_normal('')
def reply_1dpw_forum(post_url, src):
""" 加国华人网发回复函数（10个字符）
@param post_url: 帖子地址
@type post_url: str
@param src: 用户名，密码，内容，等等。
@type src: dict
@return: 是否回复成功
@rtype: bool
"""
host = utils.get_host(post_url)
logger = utils.RAPLogger(post_url)
sess = utils.RAPSession(src)
# Step 1: 登录
if not login_1dpw(post_url, sess, src):
logger.error(' Login Error')
return (False, str(logger))
logger.info(' Login OK')
resp = sess.get(post_url)
soup = BeautifulSoup(resp.content)
# 获得回复form
form = soup.find('form', attrs={'id': 'fastpostform'})
# 构造回复参数
payload = utils.get_datadic(form)
payload['message'] = src['content']
payload['posttime'] = int(time.time())
# 发送登录post包
resp = sess.post(host + form['action'] + '&inajax=1', data=payload)
# 若指定字样出现在response中，表示发帖成功
if 'Database' not in resp.content:
logger.error(' Reply Error')
return (False, str(logger))
logger.info(' Reply OK')
return (True, str(logger))
def up(ctx, recreate, rebuild, force, context=None):
if context is None:
context = ctx
host = utils.get_host(context.parent.params.get('host'))
try:
if rebuild:
workspace.build()
recreate = True
workspace = Workspace(host)
if recreate:
workspace.recreate()
sleep(1)
workspace.up()
return
except base_host.HostDownException:
if not utils.confirm_host_up(force=force, host=host):
return
except ssh_utils.SshException as e:
exit(e.returncode)
except subprocess.CalledProcessError as e:
click.echo('Unable to create the workspace', err=True)
exit(1)
ctx.invoke(up, recreate=recreate, rebuild=rebuild, force=force, context=context)
def reply_powerapple_forum(post_url, src):
# Returnable logger
logger = utils.RAPLogger(post_url)
host = utils.get_host(post_url)
sess = utils.RAPSession(src)
# Step 1: 登录
if not login_powerapple(sess, src):
logger.error(' Login Error')
return (False, str(logger))
logger.info(' Login OK')
# Step 2: Load post page
# 获取所需发帖页面，查找与{'id':'fastpostform'}匹配的form标签
resp = sess.get(post_url)
soup = BeautifulSoup(resp.content)
form = soup.find('form', attrs={'id': 'fastpostform'})
# Step 3: Submit
# 回复内容
payload = get_datadic(form)
if 'subject' in src:
payload['subject'] = src['subject']
payload['message'] = src['content']
#发送post包
resp = sess.post(host + form['action'], data=payload)
#获取回帖页面content的HTML
soup = BeautifulSoup(resp.content)
#判断回帖后页面是否含有回帖内容，若存在则证明回帖成功，否则失败
if src['content'] in resp.content:
logger.info('Reply OK')
else:
logger.error('Reply Error: please try again !')
return (False, str(logger))
return (True, str(logger))
def reply_163_bbs(post_url, src):
""" 网易论坛回复函数
- Name: 网易论坛
- Feature: bbs.*.163.com/bbs/
- Captcha: YES
- Login: YES
@param post_url: 帖子地址
@type post_url: str
@param src: 用户名，密码，回复内容，等等。
@type src: dict
@return: 是否回复成功
@rtype: bool
"""
logger = utils.RAPLogger(post_url)
sess = utils.RAPSession(src)
# Step 1: 登录
if not login_163(sess, src):
logger.error(' Login Error')
return (False, str(logger))
logger.info(' Login OK')
# Step 2: 验证码
host = utils.get_host(post_url)
logger.info(host)
page = sess.get(post_url)
# 获取各项参数
board_id = re.findall('boardId = \"(.*?)\"', page.content)[0]
thread_id = re.findall('threadId = \"(.*?)\"', page.content)[0]
# 当前时间戳
timestamp = str(time.time())
# 询问是否需要验证码
resp = sess.post(host + 'v2/post/replyCheck/' + board_id + '/' +
thread_id + '/?timestamp=' + timestamp,
headers={
'X-Requested-With': 'XMLHttpRequest',
'Origin': host,
'Referer': post_url,
'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6'
})
# 询问结果
check_code = re.findall('\"checkCode\":\"(.*?)\",', resp.content)[0]
logger.info(check_code)
seccode = ''
validate_sucess = False
post_times = 0
# 如果需要验证码
if check_code == '1':
# 询问验证码是否正确，如果不正确再次发送
while not validate_sucess \
and post_times < src['TTL']:
# 限制最大发送次数
post_times = post_times + 1
# 获取验证码图片
resp = sess.get(host + 'v2/checkcode/codeimg?timestamp=' +
timestamp,
headers={
'Accept': config.accept_image,
'Referer': post_url,
})
# 获取验证码字符串
seccode = utils.crack_captcha(resp.content)
logger.info(' seccode:' + seccode)
# 发送验证码，询问是否正确
payload = {'code': seccode.encode('utf-8')}
resp = sess.post(host + 'v2/checkcode/validate',
data=payload,
headers={
'Origin': host,
'Referer': post_url,
})
logger.info(resp.content)
validate_sucess = '"code":1' in resp.content
# Step 3: 回复
# 构造回复参数
payload = {
'checkcode': seccode,
'content': src['content'].decode('utf8').encode(CHARSET),
'title': re.findall('<title>(.*?)</title>', page.content)[0],
'boardId': board_id,
'threadId': thread_id
}
# 发送回复post包
resp = sess.post(host + 'v2/post/doReply',
data=payload,
headers={
'Referer': post_url,
})
# 若指定字样出现在response中，表示回复成功
if '\"message\">' in resp.content:
logger.error(' Reply Error ' + re.findall(
'\"message\">(.*?)</td>', resp.content.decode(CHARSET))[0])
return (False, str(logger))
logger.info(' Reply OK')
return (True, str(logger))
def test_get_host(self):
host = 'foo.appspot.com'
os.environ['HTTP_HOST'] = host
assert utils.get_host() == host
os.environ['HTTP_HOST'] = 'foo.' + host
assert utils.get_host() == host
def test_get_host(self):
host = 'foo.appspot.com'
os.environ['HTTP_HOST'] = host
assert utils.get_host() == host
os.environ['HTTP_HOST'] = 'foo.' + host
assert utils.get_host() == host
def shutdown_server():
global my_p2p_server
my_p2p_server.shutdown()
def main(my_port, c_host, c_port):
signal.signal(signal.SIGINT, signal_handler)
global my_p2p_server
my_p2p_server = ServerCore(my_port, c_host, c_port)
my_p2p_server.start()
my_p2p_server.join_network()
if __name__ == '__main__':
from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument('--c_host', default=get_host())
parser.add_argument('--c_port', default=5000, type=int)
parser.add_argument('-p',
'--port',
default=5001,
type=int,
help='port to listen on')
args = parser.parse_args()
c_host = args.c_host
c_port = args.c_port
port = args.port
main(port, c_host, c_port)
import pytest
import utils
@pytest.mark.parametrize("name", utils.get_services())
@pytest.mark.parametrize("host", utils.get_host())
def test_services(host, name):
service = host.service(name)
assert service.is_running
"""
lr = 2e-2
parser.add_argument('--lr', type=float, default=lr)
device = str('cuda:{}'.
format(gpu) if torch.cuda.is_available() and gpu != -1 else 'cpu')
parser.add_argument('--device', default=device)
num_epochs = 400
num_epochs = 2 if debug else num_epochs
parser.add_argument('--num_epochs', type=int, default=num_epochs)
"""
Other info.
"""
parser.add_argument('--user', default=get_user())
parser.add_argument('--hostname', default=get_host())
FLAGS = parser.parse_args()
COMET_EXPERIMENT = None
if FLAGS.use_comet_ml:
hyper_params = vars(FLAGS)
COMET_EXPERIMENT = Experiment(api_key=COMET_ML_APP_KEY,
project_name=COMET_PROJECT_NAME)
COMET_EXPERIMENT.log_parameters(hyper_params)
COMET_EXPERIMENT.log_parameters(model_params)
print("Experiment url, ", COMET_EXPERIMENT.url)
COMET_EXPERIMENT.add_tag(FLAGS.dataset)
for subnet in dontBlockSubnetsList:
ignoreHosts.append(subnet.subnet)
IPs = session.query(distinct(IPAddress.IP)).all()
auto_block = [
"twttr",
"akamai",
"fbcdn",
"facebook",
"1e100"
]
for IP in IPs:
host = get_host(IP[0])
if host:
host = host[0]
auto_block_bool = False
for blocker in auto_block:
if blocker in host:
print "autoblocking : {} : {}".format(IP[0], get_host(IP[0]))
auto_block_bool = True
if not auto_block_bool:
print "unknown : {} : {}".format(IP[0], get_host(IP[0]))
else:
print "could not resolve : {}".format(IP[0])
exit(-1)
def create_or_update_function(self,
name,
runtime,
handler,
description=None,
zip_filename=None,
s3_filename=None,
local_filename=None,
otherfiles=None):
if zip_filename:
zip_blob = utils.get_zip_contents(zip_filename)
code = {'ZipFile': zip_blob}
log.debug('source=zip, file=%s' % zip_filename)
elif local_filename:
zip_filename = utils.zip_file(local_filename,
otherfiles=otherfiles)
zip_blob = utils.get_zip_contents(zip_filename)
code = {'ZipFile': zip_blob}
log.debug('source=local, file=%s' % local_filename)
elif s3_filename:
bucket, key = utils.get_host(s3_filename), utils.get_path(
s3_filename)
if key.endswith('.zip'):
code = {'S3Bucket': bucket, 'S3Key': key}
else:
filename = utils.get_resource(s3_filename)
local_filename = self.download_from_s3(bucket, key, filename)
zip_filename = utils.zip_file(local_filename,
otherfiles=otherfiles)
zip_blob = utils.get_zip_contents(zip_filename)
code = {'ZipFile': zip_blob}
log.debug('source=s3, file=%s' % s3_filename)
else:
log.error('Missing source')
raise MissingSourceCodeFileError(
"Must provide either zip_filename, s3_filename or local_filename"
)
try:
_handler = '%s.%s' % (name, handler)
self.awslambda\
.update_function_configuration(FunctionName=name,
Role=self.role_arn,
Handler=_handler,
Description=description or name,
Timeout=self.timeout_time,
Runtime=runtime,
VpcConfig={
'SubnetIds': self.subnet_ids,
'SecurityGroupIds': self.security_group_ids
})
if zip_filename or local_filename:
function = self.awslambda \
.update_function_code(FunctionName=name,
ZipFile=code['ZipFile'],
Publish=True)
else:
function = self.awslambda \
.update_function_code(FunctionName=name,
S3Bucket=code['S3Bucket'],
S3Key=code['S3Key'],
Publish=True)
log.info("Lambda updated, lambda=%s" % name)
except botocore.exceptions.ClientError as ex:
if ex.response['Error']['Code'] == 'ResourceNotFoundException':
_handler = '%s.%s' % (name, handler)
# Amazon needs a few seconds to replicate the new role through
# all regions. So creating a Lambda function just after the role
# creation would sometimes result in botocore.exceptions.ClientError:
# An error occurred (InvalidParameterValueException) when calling
# the CreateFunction operation: The role defined for the function
# cannot be assumed by Lambda.
lambda_created = False
last_ex = None
for i in range(1, 10):
try:
function = self.awslambda \
.create_function(FunctionName=name,
Runtime=runtime,
Role=self.role_arn,
Handler=_handler,
Description=description or name,
Timeout=self.timeout_time,
Publish=True,
Code=code,
VpcConfig={
'SubnetIds': self.subnet_ids,
'SecurityGroupIds': self.security_group_ids
})
log.info("Lambda created, lambda=%s" % name)
lambda_created = True
break
except botocore.exceptions.ClientError as exx:
if exx.response['Error'][
'Code'] == 'InvalidParameterValueException':
log.info(
'Retrying to create lambda, lambda=%s ...' %
name)
time.sleep(3)
last_ex = exx
else:
raise exx
if not lambda_created:
raise last_ex
else:
raise ex
function_arn = function['FunctionArn']
return function_arn
def reply_sina_club(post_url, src):
""" 新浪论坛回复函数
- Name: 新浪论坛18646492184
- Feature: (forum|club).*.sina.com.cn
- Captcha: YES
- Login: YES
@param post_url: 帖子地址
@type post_url: str
@param src: 用户名，密码，回复内容，等等。
@type src: dict
@return: 是否回复成功
@rtype: bool
"""
logger = utils.RAPLogger(post_url)
sess = utils.RAPSession(src)
# Step 1: 登录
if not login_sina(sess, src):
logger.error(' Login Error')
return (False, str(logger))
logger.info(' Login OK')
# Step 2: 回复
resp = sess.get(post_url)
host = utils.get_host(post_url)
# 获取回复地址
reply_url = re.findall(r'id=\"postform\" action=\"(.*?)\"',
resp.content)[0]
soup = BeautifulSoup(resp.content)
# 获取回复form
form = soup.find('form', attrs={'id': 'postform'})
# 构造回复参数
payload = utils.get_datadic(form)
payload['message'] = src['content'].decode('utf8').encode(CHARSET)
# 替换回复地址中的特殊符号
reply_url = reply_url.replace('&', '&')
# 发送回复post包
resp = sess.post(reply_url,
data=payload,
headers={
'Origin': utils.get_host(post_url),
'Referer': post_url
})
post_times = 0
# 验证是否成功，如果失败再次发送
# 失败可能原因：验证码错误
while 'postform' not in resp.content \
and post_times < src['TTL']:
# 限制最大发送次数
post_times = post_times + 1
logger.info(' reply need captcha')
# 获取验证码图片
captcha = sess.get(host + 'seccode.php',
headers={
'Accept': config.accept_image,
'Referer': reply_url
})
# 获取验证码字符串
seccode = utils.crack_captcha(captcha.content)
logger.info(' seccode:' + seccode)
# 回复参数中增加验证码
payload['seccodeverify'] = seccode.decode(CHARSET)
# 发送回复post包
resp = sess.post(reply_url,
data=payload,
headers={
'Origin': utils.get_host(post_url),
'Referer': post_url
})
# 若指定字样出现在response中，表示回复成功
if 'postform' not in resp.content:
logger.error(' Reply Error')
return (False, str(logger))
logger.info(' Reply OK')
return (True, str(logger))
def jsonp(self):
next = request.args.get('next') or '/'
if get_host(next) != request.host:
return ''
return 'window.location.href="%s";' % next
