def create_app(config_name):
app = Flask(__name__)
app.config.from_object(configs[config_name])
gzip = Gzip(app)
app.add_url_rule('/', 'index', index)
app.add_url_rule('/<path:path>', 'index', index)
app.add_url_rule('/_nuxt/<file>', 'staticfiles', staticfiles)
api = Api(doc='/doc')
api.add_resource(DailyReport, '/api/reports/<date>/')
api.add_resource(DailyReport, '/api/reports/')
api.add_resource(Stats, '/api/stats/')
api.add_resource(Trends, '/api/trends/')
api.add_resource(Summary, '/api/summary/')
db.init_app(app)
api.init_app(app)
flask_bcrypt.init_app(app)
return app
from flask import Flask
from waitress import serve
from flask_cors import CORS
from flask_gzip import Gzip
import Config
import Models
import Controllers
import Authentication
import Workers
app = Flask(__name__, static_url_path='', static_folder='./Ui')
CORS(app, resources={r'/*': {'origins': '*'}})
Gzip(app)
Config.initialize(app)
Controllers.initialize(app)
Models.initialize(app)
Authentication.initialize(app)
Workers.initialize(app)
# print(app.url_map)
if __name__ == '__main__':
print(f'Listening on all interfaces on port 8888')
serve(app, host='0.0.0.0', port=8888)
# My libraries
from Constans import *
from Database import *
from Session import *
from Users import *
from recipe_elements.Cards import *
from recipe_elements.Recipes import *
from recipe_elements.Steps import *
from recipe_elements.Ingredients import *
from recipe_elements.Comments import *
from recipe_elements.Stars import *
from recipe_elements.Photo import *
# Global definitions
app = Flask(__name__)
gzip = Gzip(app)
global_query_id = 1000
# -----------------------------
# Utilities
def request_has_database():
return hasattr(g, 'database')
def get_database():
if not request_has_database():
Logger.info("Acquiring new database connection to server GLOBALS")
g.database = Database(global_query_id)
def create_app():
print("start app")
app = Flask(__name__)
CORS(app)
Gzip(app)
@app.route('/ping', methods=['GET'])
def ping():
return "pong"
@app.route('/api/vh/hunter', methods=['POST'])
def hunter():
id = request.headers.get('id')
article = get_json_body(request)["article"]
print("hunter", id,article)
unknow_words = Controller().find_unknow_words_by_article(id, article)
return jsonify(words=unknow_words)
@app.route('/api/vh/mark/know', methods=['POST'])
def mark_know_word():
id = request.headers.get('id')
words = request.json["words"]
Controller().mark_know_word(id, words)
return ''
@app.route('/api/vh/mark/unknow', methods=['POST'])
def mark_unknow_word():
id = request.headers.get('id')
words = request.json["words"]
Controller().mark_unknow_word(id, words)
return ''
@app.route('/api/vh/export/unknow', methods=['POST'])
def export_unknow_word():
id = request.headers.get('id')
words = Controller().get_all_unknow_word(id)
return jsonify(words=words)
@app.route('/api/vh/export/know', methods=['POST'])
def export_know_word():
id = request.headers.get('id')
words = Controller().get_all_know_word(id)
return jsonify(words=words)
@app.route('/api/vh/export/all', methods=['POST'])
def export_all_word():
print("export all")
id = request.headers.get('id')
know = Controller().get_all_know_word(id)
unknow = Controller().get_all_unknow_word(id)
return jsonify(util.to_json_serializable({"words": {"know":know, "unknow":unknow}}))
@app.route('/api/vh/import/all', methods=['POST'])
def import_all_word():
print("import_all_word start")
id = request.headers.get('id')
words = get_json_body(request)["words"]
know = words["know"]
unknow = words["unknow"]
Controller().mark_know_word(id,know)
Controller().mark_unknow_word(id,unknow)
print("import_all_word over")
return jsonify(success=True)
@app.route('/api/vh/explain', methods=['POST'])
def get_word_explain():
start_time = time.time()
id = request.headers.get('id')
print("explain", id)
words = get_json_body(request)["words"]
res = Controller().describes(id, words)
elapsed_time = time.time() - start_time
print(f"explain {len(words)} elapsed_time {elapsed_time*1000} {res}")
return jsonify(util.to_json_serializable(res))
@app.route('/api/vh/corpus', methods=['POST'])
def add_corpus():
start_time = time.time()
id = request.headers.get('id')
print("add_corpus",id)
words = get_json_body(request)
res = Controller().save_article(id, words)
elapsed_time = time.time() - start_time
print(f"collection elapsed_time {elapsed_time*1000}")
return jsonify(success=True)
return app
}
})
load_dotenv()
def getEnvVarOrDie(envVarName):
value = os.getenv(envVarName)
if not value:
raise "Must set the " + envVarName + " environment variable (have you created a .env file during local development?)"
return value
app = Flask(__name__, static_folder=None)
CORS(app)
gzip = Gzip(app, minimum_size=10)
app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
app.config['JWT_SECRET_KEY'] = getEnvVarOrDie("JWT_KEY")
app.config['JWT_BLACKLIST_ENABLED'] = True
app.config['JWT_BLACKLIST_TOKEN_CHECKS'] = ['access', 'refresh']
jwt = JWTManager(app)
schema = JsonSchema(app)
logger = app.logger
port = getEnvVarOrDie("PORT")
logger.info("Using port: %s", port)
dbUrl = getEnvVarOrDie("DATABASE_URL")
dbHostnameMatch = re.compile("^.*@([a-zA-Z0-9.:-]+)/.*$").search(dbUrl)
logger.info("Connecting to db at: %s", dbHostnameMatch.group(1))
def configure_compression(app):
app = Gzip(app)
from routing.util import path_len
from routing.bidirectional import BidirectionalAStarAnimator
from routing.astar import AStarAnimator
import os
import sys
import json
from flask import Flask, make_response, request
from flask_gzip import Gzip
from time import time
app = Flask(__name__)
gzip = Gzip(app, compress_level=9)
if "test" in sys.argv:
app.debug = True
# sys.path.append("routing/")
with open("routing/graph_data/brasilia.j") as fp:
graph = json.loads(fp.read())
with open("routing/graph_data/brasilia_coords.j") as fp:
graph_coords = json.loads(fp.read())
with open("routing/graph_data/lm_dists.j") as fp:
lm_dists = json.loads(fp.read())
BIDIRECTION = BidirectionalAStarAnimator(graph, graph_coords, lm_dists)
ANIMATOR = AStarAnimator(graph, graph_coords, lm_dists)
@app.route("/favicon.ico")
def get_favicon():
return open("static/favicon.ico", "r").read()
def create_flask_application(self):
application = Flask(__name__)
gzip = Gzip(application)
application.log = {}
application.killurl = str(uuid.uuid4())
application.jinja_env.add_extension('jinja2.ext.do')
application.config.update(
SESSION_TYPE='filesystem',
PERMANENT_SESSION_LIFETIME=60*15, # 15 min
SECRET_KEY=self.secret_key,
GITHUB_CLIENT_ID=self.client_id,
GITHUB_CLIENT_SECRET=self.client_secret,
GITHUB_CLIENT_KWARGS = {
'scope': 'repo'
}
)
Session(application)
oauth = OAuth(application)
github_bp = create_flask_blueprint(GitHub, oauth, handle_authorize)
application.register_blueprint(github_bp, url_prefix='/github')
@application.template_filter('remove_terms', )
def remove_terms(content, repository_configuration, word_boundaries=True, whole_urls=True):
"""
remove the blacklisted terms from the content
:param content: the content to anonymize
:param repository_configuration: the configuration of the repository
:return: the anonymized content
"""
repo = repository_configuration['repository']
if repo[-1] == '/':
repo = repo[0:-1]
content = re.compile("%s/blob/master" % repo, re.IGNORECASE).sub(
"%s/repository/%s" % (self.public_url, repository_configuration["id"]), content)
content = re.compile(repo, re.IGNORECASE).sub("%s/repository/%s" % (self.public_url, repository_configuration["id"]), content)
for term in repository_configuration['terms']:
if word_boundaries:
regex = re.compile(r'\b%s\b' % term, re.IGNORECASE)
else:
regex = re.compile(term, re.IGNORECASE)
if whole_urls:
def sub_url(m):
if regex.search(m.group(0)):
return 'XXX'
return m.group(0)
url_regex = re.compile('\\b((https?|ftp|file)://)[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]\\b')
content = url_regex.sub(sub_url, content)
content = regex.sub("XXX", content)
return content
@application.template_filter('file_render', )
def file_render(file, repository_configuration):
"""
produce the html representation of a file
:param file: the file to display
:param repository_configuration: the configuration of the repository
:return: the html representation of the file
"""
if type(file) == github.Commit.Commit:
return Markup(remove_terms(render_template('patch.html', patch=file), repository_configuration))
if file.type == 'dir':
return ""
if file.size > 1000000:
return Markup("The file %s is too big to be anonymized (beyond 1MB, Github limit)" % (file.name))
if ".md" in file.name or file.name == file.name.upper() or "changelog" == file.name.lower():
gh = self.github
if 'token' in repository_configuration and repository_configuration['token'] is not None:
gh = github.Github(repository_configuration['token'])
return Markup("<div class='markdown-body'>%s</div>" % remove_terms(
gh.render_markdown(file.decoded_content.decode('utf-8')),
repository_configuration))
if ".jpg" in file.name or ".png" in file.name or ".png" in file.name or ".gif" in file.name:
index = file.name.index('.')
file_extension = file.name[index + 1:]
return Markup("<img src='data:image/%s;base64, %s' alt='%s'>" % (file_extension, file.content, file.name))
if istext(file.decoded_content):
return Markup("<pre><code>{}</code></pre>")\
.format(Markup.escape(remove_terms(file.decoded_content.decode("utf-8"), repository_configuration)))
return Markup("<b>%s has an unknown extension, we are unable to anonymize it (known extensions md/txt/json/java/...)</b>" % (file.name))
@application.route('/' + application.killurl, methods=['POST'])
def seriouslykill():
func = request.environ.get('werkzeug.server.shutdown')
func()
return "Shutting down..."
def get_element_from_path(g_repo, g_commit, path):
"""
get a github element from its path
:param g_repo: the github repository
:param path: the path of the element
:return: the element
"""
if path == '':
return g_repo.get_contents('', g_commit.sha), None
current_element = os.path.basename(path)
folder_content = g_repo.get_contents(quote(os.path.dirname(path)), g_commit.sha)
for file in folder_content:
if file.name == current_element:
return file, folder_content
return None, folder_content
@application.route('/myrepo', methods=['GET'])
def myrepo():
user = session.get('user', None)
if user is None or 'token' not in user or user['token'] is None:
return redirect('github/login')
g = github.Github(user['token']['access_token'])
repos = g.get_user().get_repos(sort="full_name")
for repo in repos:
repo.uuid = str(uuid.uuid4())
return render_template('newrepo.html', repos=repos)
@application.route('/repository/<id>/commit/<sha>', methods=['GET'])
def commit(id, sha):
"""
display anonymously a commit from the repository
:param id: the repository id
:param sha: the commit id
"""
config_path = self.config_dir + "/" + str(id) + "/config.json"
if not os.path.exists(config_path):
return render_template('404.html'), 404
with open(config_path) as f:
data = json.load(f, object_hook=json_util.object_hook)
(username, repo, branch) = clean_github_repository(data['repository'])
gh = self.github
if 'token' in data:
gh = github.Github(data['token'])
g_repo = gh.get_repo("%s/%s" % (username, repo))
commit = g_repo.get_commit(sha)
return render_template('repo.html',
repository=data,
current_repository=id,
current_file=commit,
files=[],
path=[])
def is_up_to_date(repository_config, g_commit):
"""
check is the cache is up to date
:param repository_config: the repository configuration
:param g_commit: the Github commit
:return: True if the cache is up to date
"""
commit_date = datetime.strptime(g_commit.last_modified, "%a, %d %b %Y %H:%M:%S %Z")
return 'pushed_at' in repository_config and commit_date.strftime("%s") == repository_config["pushed_at"]
def get_type_content(file_name, path, repository_configuration, g_repo, is_website):
"""
Get the content type of a file from its extension
:param file_name: the filename
:param path: the path of the file
:param repository_configuration: the repository configuration
:param g_repo: the Github repository
:return: the content type
"""
if is_website:
content_type = 'text/plain; charset=utf-8'
if ".html" in file_name:
content_type = 'text/html; charset=utf-8'
if ".md" in file_name or file_name == file_name.upper():
content_type = 'text/html; charset=utf-8'
if ".jpg" in file_name \
or ".png" in file_name \
or ".gif" in file_name:
content_type = 'image/jpeg'
if ".png" in file_name:
content_type = 'image/png'
elif ".gif" in file_name:
content_type = 'image/gif'
if ".txt" in file_name \
or ".log" in file_name \
or ".csv" in file_name \
or ".xml" in file_name \
or ".json" in file_name \
or ".java" in file_name \
or ".py" in file_name \
or ".lua" in file_name \
or ".js" in file_name:
content_type = 'text/plain; charset=utf-8'
if ".xml" in file_name:
content_type = 'application/xml; charset=utf-8'
elif ".json" in file_name:
content_type = 'application/json; charset=utf-8'
elif ".js" in file_name:
content_type = 'application/javascript; charset=utf-8'
if ".css" in file_name:
content_type = 'text/css; charset=utf-8'
return content_type
return 'text/html; charset=utf-8'
def get_content(current_file, files, path, repository_config, g_repo):
"""
get the content if the page
:param current_file: the current file
:param files: the list of file of the current directory
:param path: the accessed path
:param repository_config: the repository configuration
:param g_repo: the Github repository
:return: the content of the page
"""
cache_path = os.path.join(self.config_dir, repository_config['id'], "cache")
file_path = path
if current_file is not None:
if current_file.type == 'dir':
file_path = os.path.join(current_file.path, "index.html")
else:
file_path = current_file.path
cached_file_path = os.path.join(cache_path, file_path)
content_type = get_type_content(path, path, repository_config, g_repo, False).replace("; charset=utf-8", "")
if os.path.exists(cached_file_path):
return send_from_directory(os.path.dirname(cached_file_path), os.path.basename(cached_file_path),
mimetype=content_type)
content = ''
if current_file.type != 'dir' and is_website(path, repository_config, g_repo):
if current_file.size > 1000000:
blob = g_repo.get_git_blob(current_file.sha)
if blob.encoding == 'base64':
content = base64.b64decode(blob.content).decode('utf-8')
else:
content = blob.content.decode('utf-8')
else:
content = current_file.decoded_content.decode('utf-8')
if "text" in content_type:
content = remove_terms(content, repository_config)
if ".md" in current_file.name:
gh = self.github
if 'token' in repository_config:
gh = github.Github(repository_config['token'])
content = remove_terms(gh.render_markdown(content), repository_config)
else:
tree = files
if type(tree) != list:
tree = files.tree
content = render_template('repo.html',
repository=repository_config,
current_repository=repository_config['id'],
current_file=current_file,
files=tree,
path_directory=path if type(
current_file) is not github.ContentFile.ContentFile or current_file.type == 'dir' else os.path.dirname(
current_file.path),
path=path.split("/") if path != '' else [])
content_cache_path = cached_file_path
if not os.path.exists(os.path.dirname(content_cache_path)):
os.makedirs(os.path.dirname(content_cache_path))
with open(content_cache_path, 'w') as f:
if type(content) == str:
f.write(content)
else:
f.write(content.encode('utf8'))
return content
def is_website(path, repository_config, g_repo):
"""
Check if the current request is a request to a GitHub pages
:param path: the current path
:param repository_config: the repository configuration
:param g_repo: the Github repository
:return: True if the current path is a website
"""
return path[:4] == "docs"
def is_default_file(f):
default_name = ["readme", "index"]
for name in default_name:
try:
if type(f) is github.ContentFile.ContentFile:
f.name.lower().index(name)
elif type(f) is github.GitTreeElement.GitTreeElement:
f.path.lower().index(name)
return True
except ValueError:
continue
return False
def get_current_folder_files(path, current_file, repository_config, g_repo, g_commit):
"""
get the list of files of the current repository
:param path: the path to the current file
:param current_file: the current file
:param repository_config: the repository configuration
:param g_repo: the GitHub repository
:return: the list of file of the current repository
"""
files = []
if current_file is None:
return files, current_file
if type(current_file) is not github.ContentFile.ContentFile:
files = g_repo.get_git_tree(g_commit.sha)
for f in current_file:
if is_default_file(f):
current_file = f
break
if type(current_file) is not github.ContentFile.ContentFile:
current_file = current_file[0]
elif current_file.type == 'file':
if os.path.dirname(path) == '':
files = g_repo.get_git_tree(g_commit.sha)
else:
f, folder = get_element_from_path(g_repo, g_commit, os.path.dirname(path))
if f is None:
files = folder
else:
files = g_repo.get_git_tree(f.sha)
else:
files = g_repo.get_git_tree(current_file.sha)
for f in files.tree:
if is_default_file(f):
current_file, folder = get_element_from_path(g_repo, g_commit, os.path.join(path, f.path))
break
if len(files.tree) == 1 and type(files.tree[0]) is github.ContentFile.ContentFile:
current_file, folder = get_element_from_path(g_repo, g_commit, os.path.join(path, files.tree[0].path))
return files, current_file
@application.route('/repository/<id>', methods=['GET'], defaults={'path': ''})
@application.route('/repository/<id>/', methods=['GET'], defaults={'path': ''})
@application.route('/repository/<id>/<path:path>', methods=['GET'])
@application.route('/r/<id>', methods=['GET'], defaults={'path': ''})
@application.route('/r/<id>/', methods=['GET'], defaults={'path': ''})
@application.route('/r/<id>/<path:path>', methods=['GET'])
def repository(id, path):
repo_path = self.config_dir + "/" + str(id)
config_path = repo_path + "/config.json"
if not os.path.exists(config_path):
return render_template('404.html'), 404
with open(config_path, 'r') as f:
repository_configuration = json.load(f, object_hook=json_util.object_hook)
if 'expiration_date' in repository_configuration and repository_configuration['expiration_date'] is not None:
if repository_configuration['expiration_date'] <= datetime.now(repository_configuration['expiration_date'].tzinfo):
if repository_configuration['expiration'] == 'redirect':
return redirect(repository_configuration['repository'])
elif repository_configuration['expiration'] == 'remove':
return render_template('404.html'), 404
(username, repo, branch) = clean_github_repository(repository_configuration['repository'])
gh = self.github
if 'token' in repository_configuration and repository_configuration['token'] is not None:
gh = github.Github(repository_configuration['token'])
g_commit = None
try:
g_repo = gh.get_repo("%s/%s" % (username, repo))
if branch is None:
branch = g_repo.default_branch
g_commit = g_repo.get_commit(branch)
except:
return render_template('empty.html'), 404
if not is_up_to_date(repository_configuration, g_commit):
if os.path.exists(os.path.join(repo_path, "cache")):
shutil.rmtree(os.path.join(repo_path, "cache"))
commit_date = datetime.strptime(g_commit.last_modified, "%a, %d %b %Y %H:%M:%S %Z")
repository_configuration["pushed_at"] = commit_date.strftime("%s")
with open(config_path, 'w') as fa:
json.dump(repository_configuration, fa, default=json_util.default)
cache_path = os.path.join(self.config_dir, id, "cache")
if os.path.isfile(os.path.join(cache_path, path)):
return send_from_directory(os.path.dirname(os.path.join(cache_path, path)),
os.path.basename(os.path.join(cache_path, path)),
mimetype=get_type_content(path, path, repository_configuration, g_repo, is_website(path, repository_configuration, g_repo)).replace("; charset=utf-8", ""))
elif os.path.exists(os.path.join(cache_path, path, "index.html")):
return send_from_directory(os.path.join(cache_path, path), "index.html", mimetype='text/html')
elif os.path.exists(os.path.join(cache_path, path, "README.md")):
return send_from_directory(os.path.join(cache_path, path), "README.md", mimetype='text/html')
clean_path = path
if len(clean_path) > 0 and clean_path[-1] == '/':
clean_path = clean_path[0:-1]
current_file, files = get_element_from_path(g_repo, g_commit, clean_path)
if current_file is None:
return render_template('404.html'), 404
if type(current_file) == github.ContentFile.ContentFile and current_file.type == 'dir' and len(path) > 0 and path[-1] != '/':
return redirect(url_for('repository', id=id, path=path + '/'))
files, current_file = get_current_folder_files(clean_path, current_file, repository_configuration, g_repo, g_commit)
content = get_content(current_file, files, clean_path, repository_configuration, g_repo)
content_type = get_type_content(current_file.name, clean_path, repository_configuration, g_repo, False)
return content, {'Content-Type': content_type}
@application.route('/', methods=['GET'])
def index():
id = request.args.get('id', None)
repo_name = clean_github_repository(request.args.get('githubRepository', None))
repo = None
if id is not None:
config_path = self.config_dir + "/" + id + "/config.json"
if os.path.exists(config_path):
with open(config_path) as f:
data = json.load(f, object_hook=json_util.object_hook)
if repo_name == clean_github_repository(data['repository']):
repo = data
return render_template('index.html', repo=repo)
@application.route('/robots.txt')
def robots():
return application.send_static_file('robots.txt')
@application.route('/', methods=['POST'])
def add_repository():
id = request.args.get('id', str(uuid.uuid4()))
config_path = os.path.join(self.config_dir, str(id))
repo = request.form['githubRepository']
terms = request.form['terms']
expiration_date = None
expiration = None
if 'expiration' in request.form:
expiration = request.form['expiration']
if 'expiration_date' in request.form and request.form['expiration_date'] != '':
expiration_date = datetime.strptime(request.form['expiration_date'], '%Y-%m-%d')
user = session.get('user', None)
token = None
if os.path.exists(config_path):
with open(os.path.join(config_path, "config.json"), 'r') as r:
data = json.load(r)
if 'token' in data:
token = data['token']
if token is None and user is not None and 'token' in user and user['token'] is not None:
token = user['token']['access_token']
if not os.path.exists(config_path):
os.mkdir(config_path)
with open(config_path + "/config.json", 'w') as outfile:
json.dump({
"id": id,
"repository": repo,
"terms": terms.splitlines(),
"token": token,
"expiration_date": expiration_date,
"expiration": expiration
}, outfile, default=json_util.default)
return redirect(url_for('repository', id=id))
return application
fn = 'secret.bin'
if os.path.exists(fn):
f = open(fn, 'rb');r = f.read();f.close()
else:
r = os.urandom(32)
f = open(fn, 'wb');f.write(r);f.close()
return r
app = Flask(__name__, static_url_path='')
app.config['MAX_CONTENT_LENGTH'] = 1 * 1024 * 1024
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1)
app.secret_key = get_secret()
CORS(app)
gzip = Gzip(app, minimum_size=500)
def route(r):
def rr(f):
app.add_url_rule(r, str(random.random()), f)
return rr
def calculate_etag(bin):
checksum = zlib.adler32(bin)
chksum_encoded = base64.b64encode(checksum.to_bytes(4,'big')).decode('ascii')
return chksum_encoded
# hash all resource files see if they change
def hash_these(path_arr, pattern='*.*'):
resource_files_contents = b''
for path in path_arr:
from flask import Flask, jsonify
from flask_restful import Api
from flask_gzip import Gzip
from config import PORT, ENV, config_get_current_settings_as_list
from log import error, warn, info, debug
from ibeacon_scanner.resources import ibeacon_add_http_resources_to_api
from ibeacon_scanner.services import ibeacon_init_scanner, ibeacon_stop_scanner
application = Flask(
__name__,
static_url_path='/assets',
static_folder='static',
)
flask_restful_api = Api(application)
flask_gzip = Gzip(application)
application.config['PROPAGATE_EXCEPTIONS'] = True
@application.errorhandler(404)
def page_not_found(error_msg):
response = {
"message": str(error_msg),
}
return jsonify(response), 404
@application.errorhandler(Exception)
def handle_exception(error_msg):
response = {
"message": str(error_msg),
