def index(user, id):
# FIXME: input sanitization check perms
threads_index = JsonFile("/home/kite/Maildirs/%s/threads_index.json" % user)
thread = None
for thr in threads_index.data:
if thr["id"] == id:
thread = thr
if thread == None:
abort(404, "Thread not found.")
thr["unread"] = False
threads_index.save() # FIXME: race condition here
response.content_type = "application/json"
ret_json = {"messages": [],
"subject": thread["subject"],
"date": thread["date"],
"id": thread["id"]
}
mdir = read_mail("/home/kite/Maildirs/%s" % user)
for mail_id in thread["messages"]:
ret_json["messages"].append(get_email(mdir, mail_id))
return serialize_json(ret_json)
def index(user, id):
# FIXME: input sanitization check perms
threads_index = JsonFile("/home/kite/Maildirs/%s/threads_index.json" %
user)
thread = None
for thr in threads_index.data:
if thr["id"] == id:
thread = thr
if thread == None:
abort(404, "Thread not found.")
thr["unread"] = False
threads_index.save() # FIXME: race condition here
response.content_type = "application/json"
ret_json = {
"messages": [],
"subject": thread["subject"],
"date": thread["date"],
"id": thread["id"]
}
mdir = read_mail("/home/kite/Maildirs/%s" % user)
for mail_id in thread["messages"]:
ret_json["messages"].append(get_email(mdir, mail_id))
return serialize_json(ret_json)
def __init__(self, path):
threading.Thread.__init__(self)
self.path = path
global threads_index
threads_index = JsonFile(os.path.join(self.path, "threads_index.json"))
if threads_index.data == None:
threads_index.data = {}
def send_answer(self, answer, key):
s = requests.session()
s = CacheControl(s, cache=FileCache(os.path.expanduser('~/.tst/cache')))
url = "%s/%s/answers" % (self.url, key)
data = data2json(answer).encode('utf-8')
tokens = JsonFile(os.path.expanduser('~/.tst/tokens.json'))
headers = {"Authorization": "Bearer %s" % tokens.get(self.name)}
try:
response = s.post(url, headers=headers, data=data, allow_redirects=True)
except requests.ConnectionError:
_assert(False, "Connection failed... check your internet connection (1)")
return response
def index(user):
# FIXME: input sanitization - check permissions for user
threads_index = JsonFile("/home/kite/Maildirs/%s/threads_index.json" %
user)
ret_threads = []
for thread in threads_index.data[-50:]:
ret_threads.append(thread)
response.content_type = "application/json"
return serialize_json(ret_threads)
def get(self, key):
s = requests.session()
s = CacheControl(s, cache=FileCache(os.path.expanduser('~/.tst/cache')))
url = "%s/%s" % (self.url, key)
headers = {}
tokens = JsonFile(os.path.expanduser('~/.tst/tokens.json'))
token = tokens.get(self.name)
if token:
headers['Authorization'] = 'Bearer %s' % token
try:
response = s.get(url, headers=headers, allow_redirects=True)
except requests.ConnectionError:
_assert(False, "Connection failed... check your internet connection")
if not response.ok:
self.last_error = response.status_code
self.last_response = response
return None
response.encoding = 'utf-8'
try:
resource = response.json()
resource['_response'] = response
validate_tst_object(resource)
except ValueError:
#_assert(False, "Resource is not valid json")
return None
except AssertionError as e:
print(resource)
_assert(False, "Not a TST Object: %s" % e.message)
return resource
def urls(self):
s = requests.session()
s = CacheControl(s, cache=FileCache(os.path.expanduser('~/.tst/cache')))
headers = {}
tokens = JsonFile(os.path.expanduser('~/.tst/tokens.json'))
token = tokens.get(self.name)
try:
response = s.get(self.url, allow_redirects=True)
except requests.ConnectionError:
_assert(False, "Connection failed... check your internet connection")
if not response.ok:
return None
response.encoding = 'utf-8'
try:
resource = response.json()
resource['_response'] = response
except ValueError:
return None
return resource
"""Update node."""
from typing import TYPE_CHECKING
import requests
from version import Version
from jsonfile import JsonFile
URL = "https://nodejs.org/en/"
if TYPE_CHECKING:
from .version import Version
from .jsonfile import JsonFile
install_node_version = JsonFile("./include/install/node/versions.json")
alpine_node_config = JsonFile("./containerfiles/alpine/node/config.json")
alpine_s6_node_config = JsonFile("./containerfiles/alpine-s6/node/config.json")
debian_node_config = JsonFile("./containerfiles/debian/node/config.json")
debian_s6_node_config = JsonFile("./containerfiles/debian-s6/node/config.json")
devcontainer_node_config = JsonFile("./containerfiles/devcontainer/node/config.json")
devcontainer_frontend_config = JsonFile(
"./containerfiles/devcontainer/frontend/config.json"
)
current = Version(install_node_version.read()["node"])
request = requests.get(URL).text
upstream = Version(request.split(" Current")[0].split("Download ")[-1])
if current == upstream:
print(f"Nothing to do, both current and upstream is {current}")
exit(0)
return f.read()
if __name__ == "__main__":
for site in args.sites:
logger.debug(site)
sitem = getattr(sys.modules[__name__], site)
if args.scrape:
scraper = getattr(sitem, "scraper")
html = scraper.scrape(config.min_rooms, config.max_rooms,
config.max_rent, config.wbs)
else:
scraper = None
html = get_sample(site)
parser = getattr(sitem, "parser")
flats = parser.parse(html)
jsonfile = JsonFile.open(config.jsonfile)
jsonfile.add_list(flats)
newflats = jsonfile.new_items[:]
if jsonfile.new_item_count > 0:
logging.info("Found {} new flats".format(jsonfile.new_item_count))
jsonfile.save()
if args.email and len(newflats) > 0:
sendemail.send_email(newflats, args.email)
"""Update alpine."""
from typing import TYPE_CHECKING
import requests
from version import Version
from jsonfile import JsonFile
URL = "https://registry.hub.docker.com/v2/repositories/library/alpine/tags"
skip = ["3.9", "3.9.6", "edge", "latest"]
if TYPE_CHECKING:
from .version import Version
from .jsonfile import JsonFile
alpine_base_config = JsonFile("./containerfiles/alpine/base/config.json")
alpine_s6_base_config = JsonFile("./containerfiles/alpine-s6/base/config.json")
alpine_node_config = JsonFile("./containerfiles/alpine/node/config.json")
alpine_python_config = JsonFile("./containerfiles/alpine/python/config.json")
current = Version(alpine_base_config.read()["args"]["BUILD_FROM_TAG"])
request = requests.get(URL).json()
tags = sorted(
[
x["name"]
for x in request["results"]
if x["name"][0].isdigit() and x["name"] not in skip and "." in x["name"]
]
)
upstream = Version(tags.pop())
"""Update yarn."""
from typing import TYPE_CHECKING
import requests
from version import Version
from jsonfile import JsonFile
from github import github_release
if TYPE_CHECKING:
from .version import Version
from .jsonfile import JsonFile
install_yarn_version = JsonFile("./include/install/node/versions.json")
current = Version(install_yarn_version.read()["yarn"])
upstream = Version(github_release("yarnpkg/yarn"))
if current == upstream:
print(f"Nothing to do, both current and upstream is {current}")
exit(0)
install_yarn_version.update("yarn", upstream.string)
with open("./commit", "w") as commit:
commit.write(f"Update Yarn from {current.string} to {upstream.string}")
with open("./labels", "w") as labels:
labels.write("node,debian/node,alpine/node")
"""Update ghcli."""
from typing import TYPE_CHECKING
import requests
from version import Version
from jsonfile import JsonFile
from github import github_release
if TYPE_CHECKING:
from .version import Version
from .jsonfile import JsonFile
install_ghcli_version = JsonFile("./include/install/ghcli/versions.json")
current = Version(install_ghcli_version.read()["ghcli"])
upstream = Version(github_release("cli/cli"))
if current == upstream:
print(f"Nothing to do, both current and upstream is {current}")
exit(0)
install_ghcli_version.update("ghcli", upstream.string)
with open("./commit", "w") as commit:
commit.write(
f"Update GitHub CLI from {current.string} to {upstream.string}")
with open("./labels", "w") as labels:
labels.write("devcontainer")
"""Update python."""
from typing import TYPE_CHECKING
import requests
from version import Version
from jsonfile import JsonFile
URL = "https://www.python.org/downloads"
if TYPE_CHECKING:
from .version import Version
from .jsonfile import JsonFile
install_python_version = JsonFile("./include/install/python/versions.json")
alpine_python_config = JsonFile("./containerfiles/alpine/python/config.json")
alpine_s6_python_config = JsonFile(
"./containerfiles/alpine-s6/python/config.json")
debian_python_config = JsonFile("./containerfiles/debian/python/config.json")
debian_s6_python_config = JsonFile(
"./containerfiles/debian-s6/python/config.json")
devcontainer_integration = JsonFile(
"./containerfiles/devcontainer/integration/config.json")
devcontainer_python = JsonFile(
"./containerfiles/devcontainer/python/config.json")
current = Version(install_python_version.read()["python"])
request = requests.get(URL).text
upstream = Version(request.split(">Download Python ")[2].split("<")[0])
if current == upstream:
def get_directory(self, key):
s = requests.session()
s = CacheControl(s, cache=FileCache(os.path.expanduser('~/.tst/cache')))
url = "%s/%s/tst.yaml" % (self.url, key)
headers = {}
tokens = JsonFile(os.path.expanduser('~/.tst/tokens.json'))
token = tokens.get(self.name)
if token:
headers['Authorization'] = 'Bearer %s' % token
try:
response = s.get(url, headers=headers, allow_redirects=True)
except requests.ConnectionError:
_assert(False, "Connection failed... check your internet connection (1)")
# TODO: Split method in two. The first part performs a fetch,
# while the second part (below) processes the response and
# possibly fetches further files. We could have a simple
# fetch method/funcion and a more high level get_directory
# that uses such method. In fact, the get() method above can
# also be improved with such a refactoring.
# process response
if not response.ok:
self.last_error = response.status_code
self.last_response = response
return None
response.encoding = 'utf-8'
try:
import yaml
resource = yaml.load(response.text, Loader=yaml.FullLoader)
resource['_response'] = response
except Exception as e:
cprint(YELLOW, "Failed parsing yaml: %s" % url)
cprint(YELLOW, e.message)
raise e
# gather files
files = resource.get('files') or []
files.append({
"name": "tst.yaml",
"content": response.text,
"mode": "ro"
})
## add text file if required
if 'text' in resource and is_single_line_string(resource['text']):
files.append({
"name": resource['text'],
"content": '%s/%s/%s' % (self.url, key, resource['text']),
"mode": "ro"
})
## add included files
files_filenames = [f['name'] for f in files]
for fspec in resource['include']:
filename, category, mode = parse_file_spec(fspec)
if filename not in files_filenames:
files.append({
'name': filename,
'content': '%s/%s/%s' % (self.url, key, filename),
'mode': mode
})
else:
entry = next(e for e in files if e['name'] == filename)
entry['mode'] = mode
## fetch missing files
for f in files:
if f['content'].startswith('http://') or f['content'].startswith('https://'):
f['content'] = fetch_file('%s/%s/%s' % (self.url, key, f['name']), encoding='utf-8')
return {
'kind': 'activity',
'files': files,
}
"""Update s6."""
from typing import TYPE_CHECKING
import requests
from version import Version
from jsonfile import JsonFile
from github import github_release
if TYPE_CHECKING:
from .version import Version
from .jsonfile import JsonFile
install_s6_version = JsonFile("./include/install/s6/versions.json")
current = Version(install_s6_version.read()["s6"])
upstream = Version(github_release("just-containers/s6-overlay"))
if current == upstream:
print(f"Nothing to do, both current and upstream is {current}")
exit(0)
install_s6_version.update("s6", upstream.string)
with open("./commit", "w") as commit:
commit.write(f"Update S6 from {current.string} to {upstream.string}")
with open("./labels", "w") as labels:
labels.write("S6")
"""Update debian."""
from typing import TYPE_CHECKING
import requests
from version import Version
from jsonfile import JsonFile
URL = "https://registry.hub.docker.com/v1/repositories/library/debian/tags"
skip = ["3.9", "3.9.6", "edge", "latest"]
if TYPE_CHECKING:
from .version import Version
from .jsonfile import JsonFile
debian_base_config = JsonFile("./containerfiles/debian/base/config.json")
debian_s6_base_config = JsonFile("./containerfiles/debian-s6/base/config.json")
debian_node_config = JsonFile("./containerfiles/debian/node/config.json")
debian_python_config = JsonFile("./containerfiles/debian/python/config.json")
current = Version(
debian_base_config.read()["args"]["BUILD_FROM_TAG"].split("-")[0])
request = requests.get(URL).json()
tags = [
x["name"] for x in request
if x["name"][0].isdigit() and x["name"] not in skip and "." in x["name"]
and "-slim" in x["name"] and int(x["name"].split(".")[0]) >= 10
]
upstream = Version(tags.pop().split("-")[0])