def index(user, id): # FIXME: input sanitization check perms threads_index = JsonFile("/home/kite/Maildirs/%s/threads_index.json" % user) thread = None for thr in threads_index.data: if thr["id"] == id: thread = thr if thread == None: abort(404, "Thread not found.") thr["unread"] = False threads_index.save() # FIXME: race condition here response.content_type = "application/json" ret_json = {"messages": [], "subject": thread["subject"], "date": thread["date"], "id": thread["id"] } mdir = read_mail("/home/kite/Maildirs/%s" % user) for mail_id in thread["messages"]: ret_json["messages"].append(get_email(mdir, mail_id)) return serialize_json(ret_json)
def index(user, id): # FIXME: input sanitization check perms threads_index = JsonFile("/home/kite/Maildirs/%s/threads_index.json" % user) thread = None for thr in threads_index.data: if thr["id"] == id: thread = thr if thread == None: abort(404, "Thread not found.") thr["unread"] = False threads_index.save() # FIXME: race condition here response.content_type = "application/json" ret_json = { "messages": [], "subject": thread["subject"], "date": thread["date"], "id": thread["id"] } mdir = read_mail("/home/kite/Maildirs/%s" % user) for mail_id in thread["messages"]: ret_json["messages"].append(get_email(mdir, mail_id)) return serialize_json(ret_json)
def __init__(self, path): threading.Thread.__init__(self) self.path = path global threads_index threads_index = JsonFile(os.path.join(self.path, "threads_index.json")) if threads_index.data == None: threads_index.data = {}
def send_answer(self, answer, key): s = requests.session() s = CacheControl(s, cache=FileCache(os.path.expanduser('~/.tst/cache'))) url = "%s/%s/answers" % (self.url, key) data = data2json(answer).encode('utf-8') tokens = JsonFile(os.path.expanduser('~/.tst/tokens.json')) headers = {"Authorization": "Bearer %s" % tokens.get(self.name)} try: response = s.post(url, headers=headers, data=data, allow_redirects=True) except requests.ConnectionError: _assert(False, "Connection failed... check your internet connection (1)") return response
def index(user): # FIXME: input sanitization - check permissions for user threads_index = JsonFile("/home/kite/Maildirs/%s/threads_index.json" % user) ret_threads = [] for thread in threads_index.data[-50:]: ret_threads.append(thread) response.content_type = "application/json" return serialize_json(ret_threads)
def get(self, key): s = requests.session() s = CacheControl(s, cache=FileCache(os.path.expanduser('~/.tst/cache'))) url = "%s/%s" % (self.url, key) headers = {} tokens = JsonFile(os.path.expanduser('~/.tst/tokens.json')) token = tokens.get(self.name) if token: headers['Authorization'] = 'Bearer %s' % token try: response = s.get(url, headers=headers, allow_redirects=True) except requests.ConnectionError: _assert(False, "Connection failed... check your internet connection") if not response.ok: self.last_error = response.status_code self.last_response = response return None response.encoding = 'utf-8' try: resource = response.json() resource['_response'] = response validate_tst_object(resource) except ValueError: #_assert(False, "Resource is not valid json") return None except AssertionError as e: print(resource) _assert(False, "Not a TST Object: %s" % e.message) return resource
def urls(self): s = requests.session() s = CacheControl(s, cache=FileCache(os.path.expanduser('~/.tst/cache'))) headers = {} tokens = JsonFile(os.path.expanduser('~/.tst/tokens.json')) token = tokens.get(self.name) try: response = s.get(self.url, allow_redirects=True) except requests.ConnectionError: _assert(False, "Connection failed... check your internet connection") if not response.ok: return None response.encoding = 'utf-8' try: resource = response.json() resource['_response'] = response except ValueError: return None return resource
"""Update node.""" from typing import TYPE_CHECKING import requests from version import Version from jsonfile import JsonFile URL = "https://nodejs.org/en/" if TYPE_CHECKING: from .version import Version from .jsonfile import JsonFile install_node_version = JsonFile("./include/install/node/versions.json") alpine_node_config = JsonFile("./containerfiles/alpine/node/config.json") alpine_s6_node_config = JsonFile("./containerfiles/alpine-s6/node/config.json") debian_node_config = JsonFile("./containerfiles/debian/node/config.json") debian_s6_node_config = JsonFile("./containerfiles/debian-s6/node/config.json") devcontainer_node_config = JsonFile("./containerfiles/devcontainer/node/config.json") devcontainer_frontend_config = JsonFile( "./containerfiles/devcontainer/frontend/config.json" ) current = Version(install_node_version.read()["node"]) request = requests.get(URL).text upstream = Version(request.split(" Current")[0].split("Download ")[-1]) if current == upstream: print(f"Nothing to do, both current and upstream is {current}") exit(0)
return f.read() if __name__ == "__main__": for site in args.sites: logger.debug(site) sitem = getattr(sys.modules[__name__], site) if args.scrape: scraper = getattr(sitem, "scraper") html = scraper.scrape(config.min_rooms, config.max_rooms, config.max_rent, config.wbs) else: scraper = None html = get_sample(site) parser = getattr(sitem, "parser") flats = parser.parse(html) jsonfile = JsonFile.open(config.jsonfile) jsonfile.add_list(flats) newflats = jsonfile.new_items[:] if jsonfile.new_item_count > 0: logging.info("Found {} new flats".format(jsonfile.new_item_count)) jsonfile.save() if args.email and len(newflats) > 0: sendemail.send_email(newflats, args.email)
"""Update alpine.""" from typing import TYPE_CHECKING import requests from version import Version from jsonfile import JsonFile URL = "https://registry.hub.docker.com/v2/repositories/library/alpine/tags" skip = ["3.9", "3.9.6", "edge", "latest"] if TYPE_CHECKING: from .version import Version from .jsonfile import JsonFile alpine_base_config = JsonFile("./containerfiles/alpine/base/config.json") alpine_s6_base_config = JsonFile("./containerfiles/alpine-s6/base/config.json") alpine_node_config = JsonFile("./containerfiles/alpine/node/config.json") alpine_python_config = JsonFile("./containerfiles/alpine/python/config.json") current = Version(alpine_base_config.read()["args"]["BUILD_FROM_TAG"]) request = requests.get(URL).json() tags = sorted( [ x["name"] for x in request["results"] if x["name"][0].isdigit() and x["name"] not in skip and "." in x["name"] ] ) upstream = Version(tags.pop())
"""Update yarn.""" from typing import TYPE_CHECKING import requests from version import Version from jsonfile import JsonFile from github import github_release if TYPE_CHECKING: from .version import Version from .jsonfile import JsonFile install_yarn_version = JsonFile("./include/install/node/versions.json") current = Version(install_yarn_version.read()["yarn"]) upstream = Version(github_release("yarnpkg/yarn")) if current == upstream: print(f"Nothing to do, both current and upstream is {current}") exit(0) install_yarn_version.update("yarn", upstream.string) with open("./commit", "w") as commit: commit.write(f"Update Yarn from {current.string} to {upstream.string}") with open("./labels", "w") as labels: labels.write("node,debian/node,alpine/node")
"""Update ghcli.""" from typing import TYPE_CHECKING import requests from version import Version from jsonfile import JsonFile from github import github_release if TYPE_CHECKING: from .version import Version from .jsonfile import JsonFile install_ghcli_version = JsonFile("./include/install/ghcli/versions.json") current = Version(install_ghcli_version.read()["ghcli"]) upstream = Version(github_release("cli/cli")) if current == upstream: print(f"Nothing to do, both current and upstream is {current}") exit(0) install_ghcli_version.update("ghcli", upstream.string) with open("./commit", "w") as commit: commit.write( f"Update GitHub CLI from {current.string} to {upstream.string}") with open("./labels", "w") as labels: labels.write("devcontainer")
"""Update python.""" from typing import TYPE_CHECKING import requests from version import Version from jsonfile import JsonFile URL = "https://www.python.org/downloads" if TYPE_CHECKING: from .version import Version from .jsonfile import JsonFile install_python_version = JsonFile("./include/install/python/versions.json") alpine_python_config = JsonFile("./containerfiles/alpine/python/config.json") alpine_s6_python_config = JsonFile( "./containerfiles/alpine-s6/python/config.json") debian_python_config = JsonFile("./containerfiles/debian/python/config.json") debian_s6_python_config = JsonFile( "./containerfiles/debian-s6/python/config.json") devcontainer_integration = JsonFile( "./containerfiles/devcontainer/integration/config.json") devcontainer_python = JsonFile( "./containerfiles/devcontainer/python/config.json") current = Version(install_python_version.read()["python"]) request = requests.get(URL).text upstream = Version(request.split(">Download Python ")[2].split("<")[0]) if current == upstream:
def get_directory(self, key): s = requests.session() s = CacheControl(s, cache=FileCache(os.path.expanduser('~/.tst/cache'))) url = "%s/%s/tst.yaml" % (self.url, key) headers = {} tokens = JsonFile(os.path.expanduser('~/.tst/tokens.json')) token = tokens.get(self.name) if token: headers['Authorization'] = 'Bearer %s' % token try: response = s.get(url, headers=headers, allow_redirects=True) except requests.ConnectionError: _assert(False, "Connection failed... check your internet connection (1)") # TODO: Split method in two. The first part performs a fetch, # while the second part (below) processes the response and # possibly fetches further files. We could have a simple # fetch method/funcion and a more high level get_directory # that uses such method. In fact, the get() method above can # also be improved with such a refactoring. # process response if not response.ok: self.last_error = response.status_code self.last_response = response return None response.encoding = 'utf-8' try: import yaml resource = yaml.load(response.text, Loader=yaml.FullLoader) resource['_response'] = response except Exception as e: cprint(YELLOW, "Failed parsing yaml: %s" % url) cprint(YELLOW, e.message) raise e # gather files files = resource.get('files') or [] files.append({ "name": "tst.yaml", "content": response.text, "mode": "ro" }) ## add text file if required if 'text' in resource and is_single_line_string(resource['text']): files.append({ "name": resource['text'], "content": '%s/%s/%s' % (self.url, key, resource['text']), "mode": "ro" }) ## add included files files_filenames = [f['name'] for f in files] for fspec in resource['include']: filename, category, mode = parse_file_spec(fspec) if filename not in files_filenames: files.append({ 'name': filename, 'content': '%s/%s/%s' % (self.url, key, filename), 'mode': mode }) else: entry = next(e for e in files if e['name'] == filename) entry['mode'] = mode ## fetch missing files for f in files: if f['content'].startswith('http://') or f['content'].startswith('https://'): f['content'] = fetch_file('%s/%s/%s' % (self.url, key, f['name']), encoding='utf-8') return { 'kind': 'activity', 'files': files, }
"""Update s6.""" from typing import TYPE_CHECKING import requests from version import Version from jsonfile import JsonFile from github import github_release if TYPE_CHECKING: from .version import Version from .jsonfile import JsonFile install_s6_version = JsonFile("./include/install/s6/versions.json") current = Version(install_s6_version.read()["s6"]) upstream = Version(github_release("just-containers/s6-overlay")) if current == upstream: print(f"Nothing to do, both current and upstream is {current}") exit(0) install_s6_version.update("s6", upstream.string) with open("./commit", "w") as commit: commit.write(f"Update S6 from {current.string} to {upstream.string}") with open("./labels", "w") as labels: labels.write("S6")
"""Update debian.""" from typing import TYPE_CHECKING import requests from version import Version from jsonfile import JsonFile URL = "https://registry.hub.docker.com/v1/repositories/library/debian/tags" skip = ["3.9", "3.9.6", "edge", "latest"] if TYPE_CHECKING: from .version import Version from .jsonfile import JsonFile debian_base_config = JsonFile("./containerfiles/debian/base/config.json") debian_s6_base_config = JsonFile("./containerfiles/debian-s6/base/config.json") debian_node_config = JsonFile("./containerfiles/debian/node/config.json") debian_python_config = JsonFile("./containerfiles/debian/python/config.json") current = Version( debian_base_config.read()["args"]["BUILD_FROM_TAG"].split("-")[0]) request = requests.get(URL).json() tags = [ x["name"] for x in request if x["name"][0].isdigit() and x["name"] not in skip and "." in x["name"] and "-slim" in x["name"] and int(x["name"].split(".")[0]) >= 10 ] upstream = Version(tags.pop().split("-")[0])