def do_set(self, arg):
'Sets a global parameter of WPJsonScanner'
parser = ArgumentParser(prog='set', description='sets global parameters for WPJsonScanner')
parser.add_argument("what", choices=['target', 'proxy', 'cookies', 'credentials'],
help='the parameter to set')
parser.add_argument("value", type=str, help='the new value of the parameter (for cookies, set as cookie string: "n1=v1; n2=v2")')
args = parser.custom_parse_args(arg)
if args is None:
return
if args.what == 'target':
self.target = args.value
if re.match(r'^https?://.*$', self.target) is None:
self.target = "http://" + self.target
if re.match(r'^.+/$', self.target) is None:
self.target += "/"
InteractiveShell.prompt = Console.red + self.target + Console.normal + " > "
print("target = %s" % args.value)
self.scanner = WPApi(self.target, session=self.session)
Console.log_info("Cache is erased but session stays the same (with cookies and authorization)")
elif args.what == 'proxy':
self.session.set_proxy(args.value)
print("proxy = %s" % args.value)
elif args.what == 'cookies':
self.session.set_cookies(args.value)
print("Cookies set!")
elif args.what == "credentials":
authorization_list = args.value.split(':')
if len(authorization_list) == 1:
authorization = (authorization_list[0], '')
elif len(authorization_list) >= 2:
authorization = (authorization_list[0],
':'.join(authorization_list[1:]))
self.session.set_creds(authorization)
print("Credentials set!")
print()
def create_client(self, addr):
'''
Create a client given it's address
'''
Console.print(f"[UDP] |{addr[0]}| Connected.")
client = Client(addr)
self.add_client(client)
def do_dl(self, arg):
'Downloads a media file (e.g. from /wp-content/uploads/) based on its ID'
parser = ArgumentParser(prog='dl', description='downloads a media from the server')
parser.add_argument("ids", help='ids to look for (comma separated), "all" or "cache"')
parser.add_argument("dest", help='destination folder')
parser.add_argument("--no-cache", dest="cache", action="store_false", help="don't lookup in cache and ask the server")
parser.add_argument("--use-slug", dest="slug", action="store_true", help="use the slug as filename and not the source URL name")
args = parser.custom_parse_args(arg)
if args is None:
return
if not os.path.isdir(args.dest):
Console.log_error("The destination is not a folder or does not exist")
return
print("Pulling the media URLs")
media, slugs = self.scanner.get_media_urls(args.ids, args.cache)
if len(media) == 0:
Console.log_error("No media found corresponding to the criteria")
return
print("%d media URLs found" % len(media))
answer = input("Do you wish to proceed to download? (y/N)")
if answer.lower() != "y":
return
print("Note: Only files over 10MB are logged here")
number_downloaded = 0
if args.slug:
number_downloaded = Exporter.download_media(media, args.dest, slugs)
else:
number_downloaded = Exporter.download_media(media, args.dest)
print('Downloaded %d media to %s' % (number_downloaded, args.dest))
def list_obj(self, obj_type, start, limit, is_all=False, cache=True, json=None, csv=None):
"""
Displays and exports (if relevant) the object list
:param obj_type: the type of the object
:param start: the offset of the first object
:param limit: the maximum number of objects to list
:param is_all: are all object types requested?
:param cache: whether to use the cache of not
:param json: json export filename
:param csv: csv export filename
"""
prop = self.get_fetch_or_list_type(obj_type, plural=True)
print(prop["obj_name"] + " details")
try:
kwargs = {}
if obj_type == WPApi.POST:
kwargs = {"comments": False}
obj_list = self.scanner.get_obj_list(obj_type, start, limit, cache, kwargs=kwargs)
prop["display_func"](obj_list)
InteractiveShell.export_decorator(prop["export_func"], is_all, prop["obj_name"].lower(), json, csv, obj_list)
except WordPressApiNotV2:
Console.log_error("The API does not support WP V2")
except IOError as e:
Console.log_error("Could not open %s for writing" % e.filename)
print()
def design_characters(self):
"""
Let the players design their character
"""
for player in self.player_array:
player.design()
Console.give_pc_to_next_player()
def call(traceback, warning=False):
if warning:
call_type = '[WARNING]'
else:
call_type = '[ERROR]'
Console.print('[UDP]', call_type, traceback)
def start(self):
'''
Start a infinite loop in another thread to listen to the server.
'''
self.running = True
self._thread = threading.Thread(target=self.run)
self._thread.start()
Console.print("[UDP] Started client's thread.")
def display_basic_info(information):
"""
Displays basic information about the WordPress instance
param information: information as a JSON object
"""
print()
if 'name' in information.keys():
print("Site name: %s" % html.unescape(information['name']))
if 'description' in information.keys():
print("Site description: %s" %
html.unescape(information['description']))
if 'home' in information.keys():
print("Site home: %s" % html.unescape(information['home']))
if 'gmt_offset' in information.keys():
timezone_string = ""
gmt_offset = str(information['gmt_offset'])
if '-' not in gmt_offset:
gmt_offset = '+' + gmt_offset
if 'timezone_string' in information.keys():
timezone_string = information['timezone_string']
print("Site Timezone: %s (GMT%s)" % (timezone_string, gmt_offset))
if 'namespaces' in information.keys():
print('Namespaces (API provided by addons):')
ns_ref = {}
try:
ns_ref_file = open("lib/plugins/plugin_list.csv", "rt")
ns_ref_reader = csv.reader(ns_ref_file)
for row in ns_ref_reader:
desc = None
url = None
if len(row) > 1 and len(row[1]) > 0:
desc = row[1]
if len(row) > 2 and len(row[2]) > 0:
url = row[2]
ns_ref[row[0]] = {"desc": desc, "url": url}
ns_ref_file.close()
except:
Console.log_error("Could not load namespaces reference file")
for ns in information['namespaces']:
tip = ""
if ns in ns_ref.keys():
if ns_ref[ns]['desc'] is not None:
if tip == "":
tip += " - "
tip += ns_ref[ns]['desc']
if ns_ref[ns]['url'] is not None:
if tip == "":
tip += " - "
tip += " - " + ns_ref[ns]['url']
print(' %s%s' % (ns, tip))
# TODO, dive into authentication
print()
def design(self):
"""
Ask the user how he wants his character to be
"""
self.name = Console.hello()
self.right_hand_weapon = Console.choose_weapon()
if self.right_hand_weapon.hand_status != 3:
self.left_hand_weapon = Console.choose_weapon()
def export_namespaces(namespaces, fmt, filename):
"""
**NOT IMPLEMENTED** Exports namespaces in specified format to specified file.
:param namespaces: the namespaces to export
:param fmt: the export format (JSON or CSV)
:return: the length of the list written to the file
"""
Console.log_info("Namespaces export not available yet")
return 0
def call(traceback, warning=False):
'''
Display a message in the terminal.
In the format: `"[TCP] [call type] traceback"`
'''
if warning:
call_type = "WARNING"
else:
call_type = "ERROR"
Console.print(f"[TCP] [{call_type}] {traceback}")
def call(traceback, id=None, warning=False):
if warning:
call_type = "[WARNING]"
else:
call_type = "[ERROR]"
if id == None:
Console.print("[TCP]", call_type, traceback)
else:
Console.print(f"[TCP] |{id}| {call_type} {traceback}")
def __init__(self, id_counter, conf, lock):
self.clients = []
self.id_counter = id_counter
self.conf = conf
self._lock = lock
self.console = Console(self, self._lock)
self.banned_clients = []
self.address = (socket.gethostbyname(self.conf.get_host()),
self.conf.get_port())
self.tickrate = self.conf.get_tickrate()
self.socket = socket.socket()
self.init_distributer()
def run(self):
'''
Loop that wait for connections.
Execute on_connection method.
To end execution, set running attribute to False.
'''
while self._running:
conn, addr = self._socket.accept()
Console.print(f'[TCP] |{addr[0]}| Connected.')
self.on_connection(conn, addr)
def ngAfterViewInit(self):
logger.log(self.textarea)
self._console = Console(self.textarea)
logger.log(self.namespace)
self._console.editor_ns.update(self.namespace.__dict__)
self._console.editor_ns['ns'] = self.namespace
logger.log(self._console.editor_ns)
def display_endpoints(information):
"""
Displays endpoint documentation of the WordPress API
param information: information as a JSON object
"""
print()
if 'routes' not in information.keys():
Console.log_error("Did not find the routes for endpoint discovery")
return None
for url, route in information['routes'].items():
print("%s (Namespace: %s)" % (url, route['namespace']))
for endpoint in route['endpoints']:
methods = " "
first = True
for method in endpoint['methods']:
if first:
methods += method
first = False
else:
methods += ", " + method
print(methods)
if len(endpoint['args']) > 0:
for arg, props in endpoint['args'].items():
required = ""
if props['required']:
required = " (required)"
print(" " + arg + required)
if 'type' in props.keys():
print(" type: " + str(props['type']))
if 'default' in props.keys():
print(" default: " +
str(props['default']))
if 'enum' in props.keys():
allowed = " allowed values: "
first = True
for val in props['enum']:
if first:
allowed += val
first = False
else:
allowed += ", " + val
print(allowed)
if 'description' in props.keys():
print(" " + str(props['description']))
print()
def print(self, string, *strings, warning=False):
'''
Print a string to the terminal.
'''
# compose string
for _str in strings:
string += ' ' + _str
if self.logged:
id = self.username
else:
id = self.ip
if warning:
Console.print(f'[TCP] |{id}| [WARNING] {string}')
else:
Console.print(f'[TCP] |{id}| {string}')
def display_msg(self, msg: Message):
'''
Display the Message to the terminal
in a pretty printing way.
'''
if type(msg.content) is str and '\n' in msg.content:
content = ''
elif type(msg.content) is np.ndarray:
content = ''
# player stats
elif msg.identifier == 'rpd':
content = msg.content['username']
else:
content = str(msg.content)
Console.print('[TCP] {' + msg.identifier + '} ' + content)
def fetch_obj(self, obj_type, obj_id, cache=True, json=None, csv=None):
"""
Displays and exports (if relevant) the object fetched by ID
:param obj_type: the type of the object
:param obj_id: the ID of the obj
:param cache: whether to use the cache of not
:param json: json export filename
:param csv: csv export filename
"""
prop = self.get_fetch_or_list_type(obj_type)
print(prop["obj_name"] + " details")
try:
obj = self.scanner.get_obj_by_id(obj_type, obj_id, use_cache=cache)
if len(obj) == 0:
Console.log_info(prop["obj_name"] + " not found\n")
else:
prop["display_func"](obj, details=True)
if len(prop["additional_info"].keys()) > 0:
InteractiveShell.export_decorator(prop["export_func"], False, "", json, csv, obj, prop["additional_info"])
else:
InteractiveShell.export_decorator(prop["export_func"], False, "", json, csv, obj)
except WordPressApiNotV2:
Console.log_error("The API does not support WP V2")
except IOError as e:
Console.log_error("Could not open %s for writing" % e.filename)
print()
def do_search(self, arg):
'Looks for specific keywords in the WordPress API'
parser = ArgumentParser(prog='search', description='searches something from the server')
parser.add_argument("--type", "-t", action="append", choices=[
'all',
'post',
#'post-revision',
#'wp-block',
'category',
'tag',
'page',
'comment',
'media',
'user',
#'theme',
#'search-result',
],
help='the types to look for (default all)',
dest='what'
)
parser.add_argument("keywords", help='the keywords to look for')
parser.add_argument("--json", "-j", help="list and store as json to the specified file(s)")
parser.add_argument("--csv", "-c", help="list and store as csv to the specified file(s)")
parser.add_argument("--limit", "-l", type=int, help="limit the number of results")
parser.add_argument("--start", "-s", type=int, help="start at the given index")
args = parser.custom_parse_args(arg)
if args is None:
return
what_types = WPApi.convert_obj_types_to_list(args.what)
results = self.scanner.search(what_types, args.keywords, args.start, args.limit)
print()
for k, v in results.items():
prop = self.get_fetch_or_list_type(k, plural=True)
print(prop["obj_name"] + " details")
if len(v) == 0:
Console.log_info("No result")
else:
try:
prop["display_func"](v)
InteractiveShell.export_decorator(
prop["export_func"],
len(what_types) > 1 or WPApi.ALL_TYPES in what_types,
prop["obj_name"].lower(),
args.json,
args.csv,
v
)
except WordPressApiNotV2:
Console.log_error("The API does not support WP V2")
except IOError as e:
Console.log_error("Could not open %s for writing" % e.filename)
print()
class Autobot():
def __init__(self):
self.con = Console()
self.jenkins = Jenkins('http://10.89.104.33')
def run(self, view_name, resume):
view_name = self.alias(view_name)
if self.jenkins.enable(view_name, resume):
self.con.outln(view_name + " is Enabled ", self.con.Yellow)
def alias(self, view_name):
with open('alias.json') as data_file:
data = json.load(data_file)
if data.has_key(view_name):
view_name = data[view_name]
self.con.out("Alias name found: ", self.con.White)
self.con.outln(view_name, self.con.Yellow)
return view_name
def main():
parser = argparse.ArgumentParser(
description=
"""Reads a WP-JSON API on a WordPress installation to retrieve a maximum of
publicly available information. These information comprise, but not only:
posts, comments, pages, medias or users. As this tool could allow to access
confidential (but not well-protected) data, it is recommended that you get
first a written permission from the site owner. The author won\'t endorse any
liability for misuse of this software""",
epilog=
"""(c) 2018 Mickaël "Kilawyn" Walter. This program is licensed under the MIT
license, check LICENSE.txt for more information""")
parser.add_argument('-v',
'--version',
action='version',
version='%(prog)s ' + version)
parser.add_argument('target',
type=str,
help='the base path of the WordPress installation to '
'examine')
parser.add_argument('-i',
'--info',
dest='info',
action='store_true',
help='dumps basic information about the WordPress '
'installation')
parser.add_argument('-e',
'--endpoints',
dest='endpoints',
action='store_true',
help='dumps full endpoint documentation')
parser.add_argument('-p',
'--posts',
dest='posts',
action='store_true',
help='lists published posts')
parser.add_argument('--export-posts',
dest='post_export_folder',
action='store',
help='export posts to a specified destination folder')
parser.add_argument('-u',
'--users',
dest='users',
action='store_true',
help='lists users')
parser.add_argument('-t',
'--tags',
dest='tags',
action='store_true',
help='lists tags')
parser.add_argument('-c',
'--categories',
dest='categories',
action='store_true',
help='lists categories')
parser.add_argument('-m',
'--media',
dest='media',
action='store_true',
help='lists media objects')
parser.add_argument('-g',
'--pages',
dest='pages',
action='store_true',
help='lists pages')
parser.add_argument('--export-pages',
dest='page_export_folder',
action='store',
help='export pages to a specified destination folder')
parser.add_argument('-r',
'--crawl-ns',
dest='crawl_ns',
action='store',
help='crawl all GET routes of the specified namespace '
'or all namespaces if all is specified')
parser.add_argument('-a',
'--all',
dest='all',
action='store_true',
help='dumps all available information from the '
'target API')
parser.add_argument('-S',
'--search',
dest='search',
action='store',
help='search for a string on the WordPress instance. '
'If one or several flag in agpmctu are set, search '
'only on these')
parser.add_argument('--proxy',
dest='proxy_server',
action='store',
help='define a proxy server to use, e.g. for '
'enterprise network or debugging')
parser.add_argument('--auth',
dest='credentials',
action='store',
help='define a username and a password separated by '
'a colon to use them as basic authentication')
parser.add_argument(
'--cookies',
dest='cookies',
action='store',
help='define specific cookies to send with the request '
'in the format cookie1=foo; cookie2=bar')
parser.add_argument('--no-color',
dest='nocolor',
action='store_true',
help='remove color in the output (e.g. to pipe it)')
args = parser.parse_args()
motd = """
_ _______ ___ _____
| | | | ___ \|_ | / ___|
| | | | |_/ / | | ___ ___ _ __ \ `--. ___ _ __ __ _ _ __ ___ _ __
| |/\| | __/ | |/ __|/ _ \| '_ \ `--. \/ __| '__/ _` | '_ \ / _ \ '__|
\ /\ / | /\__/ /\__ \ (_) | | | /\__/ / (__| | | (_| | |_) | __/ |
\/ \/\_| \____/ |___/\___/|_| |_\____/ \___|_| \__,_| .__/ \___|_|
| |
|_|
WPJsonScraper v%s
By Mickaël \"Kilawyn\" Walter
Make sure you use this tool with the approval of the site owner. Even if
these information are public or available with proper authentication, this
could be considered as an intrusion.
Target: %s
""" % (version, args.target)
print(motd)
if args.nocolor:
Console.wipe_color()
Console.log_info("Testing connectivity with the server")
target = args.target
if re.match(r'^https?://.*$', target) is None:
target = "http://" + target
if re.match(r'^.+/$', target) is None:
target += "/"
proxy = None
if args.proxy_server is not None:
proxy = args.proxy_server
cookies = None
if args.cookies is not None:
cookies = args.cookies
authorization = None
if args.credentials is not None:
authorization_list = args.credentials.split(':')
if len(authorization_list) == 1:
authorization = (authorization_list[0], '')
elif len(authorization_list) >= 2:
authorization = (authorization_list[0],
':'.join(authorization_list[1:]))
session = RequestSession(proxy=proxy,
cookies=cookies,
authorization=authorization)
try:
connectivity_check = session.get(target)
Console.log_success("Connection OK")
except Exception as e:
exit(0)
# Quite an ugly check to launch a search on all parameters edible
# Should find something better (maybe in argparser doc?)
if args.search is not None and not (args.all | args.posts | args.pages
| args.users | args.categories
| args.tags | args.media):
Console.log_info("Searching on all available sources")
args.posts = True
args.pages = True
args.users = True
args.categories = True
args.tags = True
args.media = True
scanner = WPApi(target, session=session, search_terms=args.search)
if args.info or args.all:
try:
basic_info = scanner.get_basic_info()
Console.log_info("General information on the target")
InfoDisplayer.display_basic_info(basic_info)
except NoWordpressApi:
Console.log_error("No WordPress API available at the given URL "
"(too old WordPress or not WordPress?)")
exit()
if args.posts or args.all:
try:
Console.log_info("Post list")
posts_list = scanner.get_all_posts()
InfoDisplayer.display_posts(posts_list)
except WordPressApiNotV2:
Console.log_error("The API does not support WP V2")
if args.pages or args.all:
try:
Console.log_info("Page list")
pages_list = scanner.get_all_pages()
InfoDisplayer.display_pages(pages_list)
except WordPressApiNotV2:
Console.log_error("The API does not support WP V2")
if args.users or args.all:
try:
Console.log_info("User list")
users_list = scanner.get_all_users()
InfoDisplayer.display_users(users_list)
except WordPressApiNotV2:
Console.log_error("The API does not support WP V2")
if args.endpoints or args.all:
try:
Console.log_info("API endpoints")
basic_info = scanner.get_basic_info()
InfoDisplayer.display_endpoints(basic_info)
except NoWordpressApi:
Console.log_error("No WordPress API available at the given URL "
"(too old WordPress or not WordPress?)")
exit()
if args.categories or args.all:
try:
Console.log_info("Category list")
categories_list = scanner.get_all_categories()
InfoDisplayer.display_categories(categories_list)
except WordPressApiNotV2:
Console.log_error("The API does not support WP V2")
if args.tags or args.all:
try:
Console.log_info("Tags list")
tags_list = scanner.get_all_tags()
InfoDisplayer.display_tags(tags_list)
except WordPressApiNotV2:
Console.log_error("The API does not support WP V2")
if args.media or args.all:
try:
Console.log_info("Media list")
media_list = scanner.get_all_media()
InfoDisplayer.display_media(media_list)
except WordPressApiNotV2:
Console.log_error("The API does not support WP V2")
if args.crawl_ns is None and args.all:
args.crawl_ns = "all"
if args.crawl_ns is not None:
try:
if args.crawl_ns == "all":
Console.log_info("Crawling all namespaces")
else:
Console.log_info("Crawling %s namespace" % args.crawl_ns)
ns_data = scanner.crawl_namespaces(args.crawl_ns)
InfoDisplayer.display_crawled_ns(ns_data)
except NSNotFoundException:
Console.log_error("The specified namespace was not found")
except Exception as e:
print(e)
if args.post_export_folder is not None:
try:
posts_list = scanner.get_all_posts()
tags_list = scanner.get_all_tags()
categories_list = scanner.get_all_categories()
users_list = scanner.get_all_users()
print()
post_number = Exporter.export_posts(posts_list,
args.post_export_folder,
tags_list, categories_list,
users_list)
if post_number > 0:
Console.log_success("Exported %d posts to %s" %
(post_number, args.post_export_folder))
except WordPressApiNotV2:
Console.log_error("The API does not support WP V2")
if args.page_export_folder is not None:
try:
pages_list = scanner.get_all_pages()
users_list = scanner.get_all_users()
print()
page_number = Exporter.export_posts(pages_list,
args.page_export_folder, None,
None, users_list)
if page_number > 0:
Console.log_success("Exported %d pages to %s" %
(page_number, args.page_export_folder))
except WordPressApiNotV2:
Console.log_error("The API does not support WP V2")
def do_request(self, method, url, data=None):
"""
Helper class to regroup requests and handle exceptions at the same
location
"""
response = None
try:
if method == "post":
response = self.s.post(url, data)
else:
response = self.s.get(url)
except requests.ConnectionError as e:
if "Errno -5" in str(e) or "Errno -2" in str(e)\
or "Errno -3" in str(e):
Console.log_error("Could not resolve host %s" % url)
raise ConnectionCouldNotResolve
elif "Errno 111" in str(e):
Console.log_error("Connection refused by %s" % url)
raise ConnectionRefused
elif "RemoteDisconnected" in str(e):
Console.log_error("Connection reset by %s" % url)
raise ConnectionReset
else:
print(e)
raise e
except Exception as e:
raise e
if response.status_code == 400:
raise HTTPError400
elif response.status_code == 401:
Console.log_error("Error 401 (Unauthorized) while trying to fetch"
" the API")
raise HTTPError401
elif response.status_code == 403:
Console.log_error("Error 403 (Authorization Required) while trying"
" to fetch the API")
raise HTTPError403
elif response.status_code == 404:
raise HTTPError404
elif response.status_code == 500:
Console.log_error("Error 500 (Internal Server Error) while trying"
" to fetch the API")
raise HTTPError500
elif response.status_code == 502:
Console.log_error("Error 502 (Bad Gateway) while trying"
" to fetch the API")
raise HTTPError404
elif response.status_code > 400:
Console.log_error("Error %d while trying to fetch the API" %
response.status_code)
raise HTTPError
return response
def __init__(self, base_url):
self.con = Console()
self.base = base_url
self.view_cache = {}
class ClientHandler:
def __init__(self, id_counter, conf, lock):
self.clients = []
self.id_counter = id_counter
self.conf = conf
self._lock = lock
self.console = Console(self, self._lock)
self.banned_clients = []
self.address = (socket.gethostbyname(self.conf.get_host()),
self.conf.get_port())
self.tickrate = self.conf.get_tickrate()
self.socket = socket.socket()
self.init_distributer()
def reset_id_counter(self):
self.id_counter = 0
def get_server_socket(self):
return self.socket
def get_console(self):
return self.console
def ban_client(self, ip):
if ip not in self.banned_clients:
self.banned_clients.append(ip)
def unban_client(self, ip):
if ip in self.banned_clients:
self.banned_clients.remove(ip)
def init_distributer(self):
self.distributer = Distributer(self.clients, self._lock, self.tickrate,
self)
client_thread = threading.Thread(target=self.distributer.distribute)
client_thread.daemon = True
client_thread.start()
def handle(self):
self.socket.bind(self.address)
self.socket.listen(5)
while True:
# Accept connections and add clients to clients list
conn, addr = self.socket.accept()
self.console.stdout("Client connected from %s:%s" %
(addr[0], str(addr[1])))
self.id_counter += 1
client = Client(conn, addr[0], addr[1], self.id_counter,
self._lock)
if client.get_ip() not in self.banned_clients:
client.sendall(Protocol().get_hello_msg(
self.tickrate, client.get_id()))
self.clients.append(client)
client_listen_thread = threading.Thread(target=self._listen,
args=[client])
client_listen_thread.daemon = True
client_listen_thread.start()
def _listen(self, client):
while True:
try:
if client.get_ip() in self.banned_clients:
raise Exception
client.recv()
except Exception as e:
self.console.stdout(
"Client {}:{} has been disconnected.".format(
client.get_ip(), client.get_port()))
self.clients.remove(client)
break
def __init__(self):
self.con = Console()
self.jenkins = Jenkins('http://10.89.104.33')
def do_request(self, method, url, data=None):
"""
Helper class to regroup requests and handle exceptions at the same
location
"""
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.146 Safari/537.36'
}
response = None
try:
if method == "post":
response = self.s.post(url, data, headers=headers)
else:
response = self.s.get(url, data=None, header=headers)
except requests.ConnectionError as e:
if "Errno -5" in str(e) or "Errno -2" in str(e)\
or "Errno -3" in str(e):
Console.log_error("Could not resolve host %s" % url)
raise ConnectionCouldNotResolve
elif "Errno 111" in str(e):
Console.log_error("Connection refused by %s" % url)
raise ConnectionRefused
elif "RemoteDisconnected" in str(e):
Console.log_error("Connection reset by %s" % url)
raise ConnectionReset
else:
print(e)
raise e
except Exception as e:
raise e
if response.status_code == 400:
raise HTTPError400
elif response.status_code == 401:
Console.log_error("Error 401 (Unauthorized) while trying to fetch"
" the API")
raise HTTPError401
elif response.status_code == 403:
Console.log_error("Error 403 (Authorization Required) while trying"
" to fetch the API")
raise HTTPError403
elif response.status_code == 404:
raise HTTPError404
elif response.status_code == 500:
Console.log_error("Error 500 (Internal Server Error) while trying"
" to fetch the API")
raise HTTPError500
elif response.status_code == 502:
Console.log_error("Error 502 (Bad Gateway) while trying"
" to fetch the API")
raise HTTPError404
elif response.status_code > 400:
Console.log_error("Error %d while trying to fetch the API" %
response.status_code)
raise HTTPError
return response
def __init__(self):
self.__console = Console()
self.__frameset_size = (64, 85, 4)
self.__frameset = None
self.__last_actions = None
self.__hot_encode_action_size = (4, 4)
class Environment:
def __init__(self):
self.__console = Console()
self.__frameset_size = (64, 85, 4)
self.__frameset = None
self.__last_actions = None
self.__hot_encode_action_size = (4, 4)
@property
def frameset_size(self):
return self.__frameset_size
@property
def hot_encode_action_size(self):
return self.__hot_encode_action_size
def hot_encode_action(self, action):
# bit 1: JUMP
# bit 2: FIRE/ACC
# bit 3: RIGHT
# but 4: LEFT
if action == 0: # RIGHT
return np.array([
0,
0,
1,
0,
])
if action == 1: # RIGHT + FIRE
return np.array([
0,
1,
1,
0,
])
if action == 2: # RIGHT + JUMP
return np.array([
1,
0,
1,
0,
])
if action == 3: # RIGHT + JUMP + FIRE
return np.array([
1,
1,
1,
0,
])
if action == 4: # LEFT
return np.array([
0,
0,
0,
1,
])
def start(self):
_ = self.__console.recv()
def exit(self):
self.__console.close()
def sendAction(self, action):
self.__console.send('{}\n'.format(action))
def getState(self):
buf = self.__console.recv()
feedback = buf.split(' ')
reward = int(feedback[0])
done = int(feedback[1])
level_position = int(feedback[2])
action = self.hot_encode_action(int(feedback[3]))
# get screenshot as a [192,256,3] array with the last dimension representing RGB
# this is the actual resolution of the game in the emulator
# no need to do any extra processing
img = ImageGrab.grabclipboard()
# just a security because from time to time, img may be None...
while img is None:
img = ImageGrab.grabclipboard()
img = img.convert('L')
img = img.resize((85, 64), resample=ANTIALIAS)
# img.save(f'{random.random()}.png')
# if self.__frameset is None:
# img_array = np.array(img)
# self.__frameset = np.stack([img_array, img_array, img_array, img_array])
# else:
# last_screenshots = self.__frameset[1:,:,:]
# new_screenshot = np.array(img)
# self.__frameset = np.insert(last_screenshots, 3, new_screenshot, axis=0)
if self.__frameset is None:
img_array = np.array(img)
self.__frameset = np.stack(
[img_array, img_array, img_array, img_array], axis=2)
else:
img_array = np.array(img)
self.__frameset = np.append(self.__frameset[:, :, 1:],
np.stack([img_array], axis=2),
axis=2)
if self.__last_actions is None:
self.__last_actions = np.stack([action, action, action, action])
else:
actions = self.__last_actions[1:]
self.__last_actions = np.insert(actions, 3, action, axis=0)
return (reward, self.__frameset, done, level_position,
self.__last_actions)
class Jenkins():
def __init__(self, base_url):
self.con = Console()
self.base = base_url
self.view_cache = {}
def enable(self, view_name, do_not_enable_child):
if do_not_enable_child:
self.con.out(
"Enabling view " + view_name +
" and resuming all enabled children...", self.con.White)
else:
self.con.out("Enabling view " + view_name + " and all children...",
self.con.White)
if self.trigger_post(self.base + '/job/' + view_name + '/enable'):
if not do_not_enable_child:
tasks = self.view(view_name)
for t in tasks:
self.con.out("- Enabling job " + t + "...", self.con.White)
self.trigger_post(self.base + '/job/' + t + '/enable')
return True
return False
def wait_until_task_done(self, task_name, build_num):
self.con.out(".", self.con.White)
while not self.task_done(task_name, build_num):
time.sleep(2)
self.con.out(".", self.con.White)
def task_done(self, task_name, build_num):
result = self.get(self.base + '/job/' + task_name + '/api/json')
if result['nextBuildNumber'] == build_num:
return False
if result['inQueue']:
return False
if result['color'] in ['blue', 'red']:
return True
return False
def build(self, view_name):
self.con.outln("Building view " + view_name + "...", self.con.White)
tasks = self.view(view_name)
retry_tasks = []
critical_fail = []
retry_max = 3
retry_count = 0
while len(tasks) > 0 and retry_count <= retry_max:
for t in tasks:
result = self.get(self.base + '/job/' + t + '/api/json')
if result['color'] != 'disabled':
nextBuildNumber = result['nextBuildNumber']
self.con.out(
"- Building job " + t + " #" + str(nextBuildNumber) +
"...", self.con.White)
if self.trigger_post(self.base + '/job/' + t + '/build',
False):
self.wait_until_task_done(t, nextBuildNumber)
stat = self.grep_stat(t, nextBuildNumber)
fail_count = stat.attrib['fail']
pass_count = stat.attrib['pass']
if fail_count == "0":
self.con.outln(
"PASSED [" + pass_count + "/" + pass_count +
"]", self.con.Green)
self.con.out("--= Disabling job " + t + "...",
self.con.White)
self.trigger_post(self.base + '/job/' + t +
'/disable')
else:
self.con.outln(
"FAILED [" + pass_count + "/" +
str(int(pass_count) + int(fail_count)) + "]",
self.con.Red)
should_retry = self.analyze_error_log(
t, nextBuildNumber)
if should_retry:
retry_tasks.append(t)
else:
critical_fail = critical_fail.append(t)
else:
self.con.outln(
"- Skipping job " + t + " due to job is disabled...",
self.con.White)
if len(retry_tasks) > 0:
retry_count += 1
if retry_count <= retry_max:
self.con.outln(
"Retry #" + str(retry_count) + " : " +
str(len(retry_tasks)) + " task(s)", self.con.White)
tasks = retry_tasks
retry_tasks = []
else:
tasks = []
if len(tasks) == 0 and len(critical_fail) == 0:
self.con.outln("ALL TEST PASSED", self.con.Green)
self.con.out("- Running clean up task for " + view_name + "...",
self.con.White)
result = self.get(self.base + '/job/' + view_name + '/api/json')
nextBuildNumber = result['nextBuildNumber']
if self.trigger_post(self.base + '/job/' + view_name + '/build',
False):
self.wait_until_task_done(view_name, nextBuildNumber)
self.con.outln("OK", self.con.Green)
self.con.out("- Disabling view " + view_name + "...",
self.con.White)
self.trigger_post(self.base + '/job/' + view_name + '/disable')
def analyze_error_log(self, task_name, build_num):
return True
def grep_stat(self, task_name, build_num):
result = self.get(self.base + '/job/' + task_name + '/' +
str(build_num) + '/robot/report/output.xml')
stat = result.findall('./statistics/total/stat')
return stat[-1]
def view(self, view_name):
if view_name not in self.view_cache or self.view_cache[
view_name] == None:
result = self.get(self.base + '/job/' + view_name + '/api/json')
if result != None:
out = []
for p in result['downstreamProjects']:
out.append(p['name'])
self.view_cache[view_name] = out
return out
else:
self.view_cache[view_name] = None
return self.view_cache[view_name]
def trigger_get(self, url, verbose=False):
resp = requests.get(url)
if resp.status_code == 200 or resp.status_code == 201:
if verbose:
self.con.outln("OK", self.con.Green)
return True
else:
if verbose:
self.con.outln("FAILED", self.con.Red)
return False
def trigger_post(self, url, verbose=True):
resp = requests.post(url)
if resp.status_code == 200 or resp.status_code == 201:
if verbose:
self.con.outln("OK", self.con.Green)
return True
else:
if verbose:
self.con.outln("FAILED", self.con.Red)
return False
def get(self, url):
resp = requests.get(url)
content_type = resp.headers['content-type'].split(';')[0]
if resp.status_code == 200 or resp.status_code == 201:
if content_type in ['application/json', 'text/json']:
js = json.loads(resp.content)
return js
elif content_type in ['application/xml']:
root = ET.fromstring(resp.content)
return root
return None
def get_config(self, view_name):
resp = self.get(self.base + '/job/' + view_name + '/config.xml')
return resp