def updateItem(item_name):
editItem = session.query(Item).filter_by(name=item_name).one()
if login_session['user_id'] != editItem.user_id:
return "<script>function myFunction() {alert('You are not authorized to edit this item, as it is not yours.');}</script><body onload='myFunction()''>"
if request.method == 'POST':
if request.form['name']:
editItem.name = request.form['name']
if request.form['description']:
editItem.description = request.form['description']
if request.form['category_name']:
category = session.query(Category).filter_by(name=request.form['category_name']).one()
editItem.category = category
editItem.category_id = category.id
file = request.files['file']
if file and allowed_file(file.filename):
print 'updating files...'
filename = secure_filename(file.filename)
# delete old file if it exists
if (os.path.isfile(safe_join(app.config['UPLOAD_FOLDER'], str(editItem.user_id) + "_" + editItem.picture))):
print 'deleting old file...'
os.remove(safe_join(app.config['UPLOAD_FOLDER'], str(editItem.user_id) + "_" + editItem.picture))
# save new file
file.save(safe_join(app.config['UPLOAD_FOLDER'], str(editItem.user_id) + "_" + filename))
editItem.picture = filename
print 'updated file!'
session.add(editItem)
session.commit()
return redirect(url_for('item', category_name=request.form['category_name'], item_name=editItem.name))
else:
categories = session.query(Category)
return render_template('updateItem.html', item=editItem, categories=categories)
def save_database_as_zip():
app = flask.current_app
if not app.config.get("DEBUG", False):
flask.abort(404)
database_dump = dump_database(auth=False).data
user_photos_dump = dump_user_photos(auth=False)
# create temporary files
temp = ppath(tempfile.mkdtemp())
database_path = flask.safe_join(temp, "database")
os.makedirs(database_path)
with open(flask.safe_join(database_path, "dump.json"), "w+") as f:
f.write(database_dump)
with open(flask.safe_join(database_path, "user_photos.zip"), "w+") as f:
f.write(user_photos_dump.data)
archive_name = flask.safe_join(temp, "database")
try:
shutil.make_archive(archive_name, "zip", database_path)
f = open(archive_name + ".zip")
finally:
temp.rmtree()
return flask.Response(sugar.read_file(f), mimetype="application/x-zip-compressed")
def site_show(page):
if page.endswith('.html'):
return redirect(url_for('site_show', page=page[:-5]))
# Redirect for old spec pages
if page.startswith('docs/spec/'):
return redirect(url_for('spec_show', name=page[10:]))
if page in SPEC_REDIRECTS:
return redirect(url_for('spec_show', name=SPEC_REDIRECTS[page]))
name = 'site/%s.html' % page
page_file = safe_join(TEMPLATE_DIR, name)
if not os.path.exists(page_file):
# Could be a directory, so try index.html
name = 'site/%s/index.html' % page
page_file = safe_join(TEMPLATE_DIR, name)
if not os.path.exists(page_file):
# bah! those damn users all the time!
abort(404)
options = {
'page': page,
}
if (page == 'index'):
options['blog_posts'] = get_blog_posts(8)
# hah!
return render_template(name, **options)
def restore(dump_name):
""" Restore page from dump storage """
if not dump_name or dump_name == '.gitignore' or '.' in dump_name or '/' in dump_name:
abort(404)
if '@' in dump_name:
dump = dump_name.split('@')
page_file = safe_join(app.config['PAGES_FOLDER'], dump[0])
dump_file = safe_join(app.config['DUMPS_FOLDER'], dump[0] + '@' + dump[1])
try:
dump_page(dump[0])
shutil.copyfile(dump_file, page_file)
flash('Success!', 'info')
except OSError:
flash('Can not restore this page!', 'error')
finally:
return redirect(url_for('page', page_name=dump[0]))
dumps = []
timestamps = show_dumps(dump_name)
for timestamp in timestamps:
hr_time = datetime.datetime.fromtimestamp(int(timestamp)).strftime('%d-%m-%Y %H:%M')
dump = {'timestamp': timestamp, 'hr_time': hr_time}
dumps.append(dump)
if not dumps:
flash('Dumps for this page not found!', 'error')
return redirect(url_for('page', page_name=dump_name))
return render_template('restore.html', context={"title": dump_name, "dumps": dumps})
def subdir_handler(filename):
path = safe_join(root, subdir)
path = safe_join(path, filename)
if os.path.exists(path):
return send_file(path)
else:
flask.abort(404)
def upload():
""" File uploading handler. """
url = request.args.get('url', None)
delete = request.args.get('delete', None)
if request.method == 'POST':
file = request.files['file']
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
file.save(safe_join(app.config['UPLOAD_FOLDER'], filename))
flash('File ' + filename + ' was uploaded!', 'info')
else:
flash('File was not uploaded!', 'error')
if url is not None:
file = file_from_url(url)
if file and allowed_file(file['name']):
file_name = secure_filename(file['name'])
file_path = safe_join(app.config['UPLOAD_FOLDER'], file_name)
with open(file_path, 'wb') as f:
f.write(file['content'])
flash('File ' + file_name + ' was uploaded!', 'info')
else:
flash('Can not download file!', 'error')
if delete is not None:
file = safe_join(app.config['UPLOAD_FOLDER'], delete)
try:
os.remove(file)
flash('Success!', 'info')
except FileNotFoundError:
flash('File not found!', 'error')
except Exception:
abort(500)
files = show_files()
return render_template('upload.html', context={"files": files})
def serve_page(page): if os.path.isdir(safe_join(SITE_DIR, page)): # Serve the index page page_file = safe_join(page, "index.html") else: # Serve the specific page page_file = "%s.html" % page return send_from_directory(SITE_DIR, page_file)
def dump_page(page_name=None):
""" Backup current page to <dumps_path> directory """
dumps_list = show_dumps(page_name)
if len(dumps_list) > 9:
os.remove(app.config['DUMPS_FOLDER'] + page_name + '@' + dumps_list[0])
page_file = safe_join(app.config['PAGES_FOLDER'], page_name)
stamp_file = safe_join(app.config['DUMPS_FOLDER'], page_name + '@' + str(int(time.time()))) # hrr
shutil.copyfile(page_file, stamp_file)
def get_repo_names():
children = (
(name, safe_join(current_app.config['RESTFULGIT_REPO_BASE_PATH'], name))
for name in os.listdir(current_app.config['RESTFULGIT_REPO_BASE_PATH'])
)
subdirs = [(dir_name, full_path) for dir_name, full_path in children if os.path.isdir(full_path)]
mirrors = set(name for name, _ in subdirs if name.endswith('.git'))
working_copies = set(name for name, full_path in subdirs if os.path.isdir(safe_join(full_path, '.git')))
repositories = mirrors | working_copies
return repositories
def admin_review():
if not g.permission_reviewer:
abort(401)
if request.method == 'POST':
username = request.form['username']
filename = secure_filename(request.form['document'])
src_path = safe_join(safe_join(app.config['DOCUMENTS_LOCATION'], 'pending'), username)
src = safe_join(src_path, filename)
dst_path = safe_join(safe_join(app.config['DOCUMENTS_LOCATION'], 'public'), username)
dst = safe_join(dst_path, filename)
if not os.path.exists(dst_path):
os.makedirs(dst_path)
if not os.path.exists(src):
flash('Document not found. Is it hiding in a closet, or are you meesing with the system?', error)
else:
os.rename(src, dst)
flash('Document published.')
g.documents = []
users = os.listdir(safe_join(app.config['DOCUMENTS_LOCATION'], 'pending'))
for user in users:
user_documents = os.listdir(safe_join(safe_join(app.config['DOCUMENTS_LOCATION'], 'pending'), user))
for user_document in user_documents:
g.documents.append((user, user_document))
return render_template('admin_review.html')
def get_repo_list():
children = (
(name, safe_join(current_app.config['RESTFULGIT_REPO_BASE_PATH'], name))
for name in os.listdir(current_app.config['RESTFULGIT_REPO_BASE_PATH'])
)
subdirs = [(dir_name, full_path) for dir_name, full_path in children if os.path.isdir(full_path)]
mirrors = set(name for name, _ in subdirs if name.endswith('.git'))
working_copies = set(name for name, full_path in subdirs if os.path.isdir(safe_join(full_path, '.git')))
repositories = list(mirrors | working_copies)
repositories.sort()
flask.g.etag = hashlib.sha1(''.join(repositories)).hexdigest()
return {'repos': repositories}
def _filename(self, **kwargs):
if 'session' in kwargs:
session=kwargs['session']
elif 'cell_id' in kwargs:
session = kwargs['cell_id']
else:
session = "SESSION_NOT_FOUND"
session_subdir = list(str(session)[:self._levels])+[str(session)]
# Use Flask's safe_join to make sure we don't overwrite something crucial
session_dir = safe_join(self._dir, os.path.join(*session_subdir))
if not os.path.isdir(session_dir):
os.makedirs(session_dir)
return safe_join(session_dir, kwargs['filename'])
def save_file(reqfile, path):
reqfile_path = safe_join(path, secure_filename(reqfile.filename))
reqfile.save(reqfile_path)
with file(reqfile_path) as __f:
ghash, length = githash(__f.read())
new_dir = safe_join(path, ghash[0:2])
new_filename = ghash[2:40]
new_path = safe_join(new_dir, new_filename)
try:
os.makedirs(new_dir)
except OSError:
pass # dir already exists: OK
os.rename(reqfile_path, new_path)
return safe_join(ghash[0:2], ghash[2:40]), length
def subtitles():
"""Re-encode subtitles to UTF-8 for scandinavian letters to work"""
movie = request.args.get('movie')
videofolder = VIDEO_FOLDER
path = safe_join(videofolder, movie)
if movie.endswith('.srt'):
filename = safe_join(videofolder, movie)
with open(filename, 'r') as fd:
movie = fd.read()
movie = movie.decode('iso-8859-1').encode('utf-8')
return Response(movie, mimetype='text/plain')
else:
flash('Not a subtitle file.')
return redirect(url_for('index'))
def make_thumb(self, filename, miniature=None, override=False, size='96x96',
width=None, height=None, crop=None, bg=None, quality=85):
"""
生成缩略图
:param filename: 图像源文件名
:param miniature: 缩略图文件名，如果为None则按照参数自动生成
:param override: 是否覆盖同名文件
:param size: 缩略图尺寸，当width和height参数之一为None时生效
:param width: 缩略图宽度
:param height: 缩略图高度
:param crop: 是否需要裁剪
:param bg: 背景颜色
:param quality: 图像压缩质量
"""
if not width or not height:
width, height = [int(x) for x in size.split('x')]
name, fm = os.path.splitext(filename)
if not miniature:
miniature = self._get_name(name, fm, size, crop, bg, quality)
thumb_filename = flask.safe_join(self.config.thumb_destination, miniature)
self._ensure_path(thumb_filename)
if not os.path.exists(thumb_filename) or override:
original_filename = flask.safe_join(self.config.destination, filename)
if not os.path.exists(original_filename):
return None
thumb_size = (width, height)
try:
image = Image.open(original_filename)
except IOError:
return None
if crop == 'fit':
img = ImageOps.fit(image, thumb_size, Image.ANTIALIAS)
else:
img = image.copy()
img.thumbnail((width, height), Image.ANTIALIAS)
if bg:
img = self._bg_square(img, bg)
img.save(thumb_filename, image.format, quality=quality)
return miniature
def deleteItem(item_name):
itemToDelete = session.query(Item).filter_by(name=item_name).one()
if login_session['user_id'] != itemToDelete.user_id:
return "<script>function myFunction() {alert('You are not authorized to delete this item, as it is not yours.');}</script>" \
+ "<body onload='myFunction()''>"
if request.method == 'POST':
category_name = itemToDelete.category.name
# delete file if it exists
if (itemToDelete.picture is not None and os.path.isfile(safe_join(app.config['UPLOAD_FOLDER'], str(itemToDelete.user_id) + "_" + itemToDelete.picture))):
os.remove(safe_join(app.config['UPLOAD_FOLDER'], str(itemToDelete.user_id) + "_" + itemToDelete.picture))
session.delete(itemToDelete)
session.commit()
return redirect(url_for('category', category_name=category_name))
else:
return render_template('deleteItem.html', item=itemToDelete)
def files_get_sha256_helper(sha256, raw='f'):
'''
Returns binary from storage. Defaults to password protected zipfile.
'''
file_path = safe_join(api_config['api']['upload_folder'], sha256)
if not os.path.exists(file_path):
abort(HTTP_NOT_FOUND)
with open(file_path, 'rb') as fh:
fh_content = fh.read()
raw = str(raw)[0].lower()
if raw in ['t', 'y', '1']:
response = make_response(fh_content)
response.headers['Content-Type'] = 'application/octet-stream; charset=UTF-8'
# better way to include fname?
response.headers['Content-Disposition'] = 'inline; filename={}.bin'.format(sha256)
else:
# ref: https://github.com/crits/crits/crits/core/data_tools.py#L122
rawname = sha256 + '.bin'
with open(safe_join('/tmp/', rawname), 'wb') as raw_fh:
raw_fh.write(fh_content)
zipname = sha256 + '.zip'
args = ['/usr/bin/zip', '-j',
safe_join('/tmp', zipname),
safe_join('/tmp', rawname),
'-P', 'infected']
proc = subprocess.Popen(args)
wait_seconds = 30
while proc.poll() is None and wait_seconds:
time.sleep(1)
wait_seconds -= 1
if proc.returncode:
return make_response(jsonify({'Error': 'Failed to create zip ()'.format(proc.returncode)}))
elif not wait_seconds:
proc.terminate()
return make_response(jsonify({'Error': 'Process timed out'}))
else:
with open(safe_join('/tmp', zipname), 'rb') as zip_fh:
zip_data = zip_fh.read()
if len(zip_data) == 0:
return make_response(jsonify({'Error': 'Zip file empty'}))
response = make_response(zip_data)
response.headers['Content-Type'] = 'application/zip; charset=UTF-8'
response.headers['Content-Disposition'] = 'inline; filename={}.zip'.format(sha256)
return response
def render(filename=None):
if filename is not None:
filename = safe_join(directory, filename)
if os.path.isdir(filename):
try:
filename = _find_file(filename)
except ValueError:
abort(404)
try:
text = _read_file(filename)
except IOError as ex:
if ex.errno != errno.ENOENT:
raise
return abort(404)
# if we think this file is an image, serve it as such
mimetype, _ = mimetypes.guess_type(filename)
if mimetype.startswith("image/"):
return render_image(text, mimetype)
filename_display = _display_filename(filename)
else:
text = _read_file(path)
filename_display = _display_filename(path)
return render_page(text, filename_display, gfm, context, render_offline,
username, password, style_urls)
def create_user(username=None, password=None):
""" Create user """
user_file = safe_join(app.config['USERS_FOLDER'], username)
password = sha256_crypt.encrypt(password)
with open(user_file, 'x') as f:
dump({'password': password}, f)
settings_write(username, 'create', int(time.time()))
def settings_write(name=None, key=None, value=None):
""" Write to settings file """
file = safe_join(app.config['SETTINGS_FOLDER'], name + '.json')
content = settings_read(name)
content[key] = value
with open(file, 'w') as f:
dump(content, f)
def render(filename=None):
if filename is not None:
filename = safe_join(os.path.dirname(app.config['GRIP_FILE']), filename)
if os.path.isdir(filename):
try:
filename = _find_file(filename)
except ValueError:
abort(404)
# if we think this file is an image, we need to read it in
# binary mode and serve it as such
mimetype, _ = mimetypes.guess_type(filename)
is_image = mimetype.startswith('image/') if mimetype else False
try:
text = _read_file(filename, is_image, encoding)
except IOError as ex:
if ex.errno != errno.ENOENT:
raise
return abort(404)
if is_image:
return render_image(text, mimetype)
else:
filename = app.config['GRIP_FILE']
text = _read_file(app.config['GRIP_FILE'])
return render_page(text, filename, gfm, context,
username, password, render_offline, style_urls, styles)
def login():
""" Login handler. Check password from user file and add <username> to session storage. """
if request.method == 'POST':
username = escape(request.form.get('username', None))
password = request.form.get('password', None)
next_url = request.args.get('next', url_for('main'))
if not username or not password:
flash('Fill all fields!', 'error')
return redirect(url_for('login', next=next_url))
try:
user_file = safe_join(app.config['USERS_FOLDER'], username)
with open(user_file, 'r') as uf:
user_conf = load(uf) # user_file on json format
if not sha256_crypt.verify(password, user_conf['password']): # check password
flash('Wrong password!', 'error')
return redirect(url_for('login', next=next_url))
else:
flash('You successfully logged in!', 'info')
session['username'] = username
settings_write(username, 'last_login', int(time.time()))
except FileNotFoundError:
flash('User not exist!', 'error')
return redirect(url_for('login', next=next_url))
except Exception:
abort(500)
return redirect(next_url)
return render_template('login.html')
def run_test(chapter_num=None, section_num=None, test_index=None,
test_id=None):
if test_id is None:
try:
chapter, sections, _ = chapters[chapter_num - 1]
title, url, tests = sections[section_num - 1]
test = tests[test_index - 1]
previous_index = test_index - 1
next_index = test_index + 1 if test_index < len(tests) else None
except IndexError:
abort(404)
else:
test = dict(test_id=test_id)
from pygments import highlight
from pygments.lexers import HtmlLexer
from pygments.formatters import HtmlFormatter
filename = safe_join(suite_directory, test['test_id'] + '.htm')
with open(filename, 'rb') as fd:
source = fd.read().decode('utf8')
formatter = HtmlFormatter(linenos='inline')
source = highlight(source, HtmlLexer(), formatter)
css = formatter.get_style_defs('.highlight')
return render_template('run_test.html', **locals())
def render_pdf(template, temp, height, width, context={}, filename=None, **kwargs):
filename = filename or random_string()
is_celery_processs = kwargs.pop('is_celery_processs', None)
use_wkhtmltopdf = kwargs.pop('use_wkhtmltopdf', None)
# save badge as html
html_path = save_template(template, directory=temp, context=context,
filename=filename + ".html")
# save badge as pdf
pdf_path = flask.safe_join(temp, filename + ".pdf")
def generate_pdf():
with open(flask.safe_join(temp, "stderr.log"), "w+") as stderr:
if use_wkhtmltopdf:
wkhtmltopdf(html_path=html_path, pdf_path=pdf_path,
height=height, width=width, stderr=stderr)
else:
pisa_magic(html_path=html_path, pdf_path=pdf_path,
height=height, width=width, stderr=stderr)
pdf = open(pdf_path, "rb")
return pdf
if is_celery_processs:
generate_pdf()
return pdf_path.name
else:
try:
pdf = generate_pdf()
finally:
temp.rmtree()
return flask.Response(read_file(pdf), mimetype="application/pdf")
def render(filename=None):
if filename is not None:
filename = safe_join(os.path.dirname(in_filename), filename)
if os.path.isdir(filename):
filename = _find_file_or_404(filename, force_resolve)
# Read and serve images as binary
mimetype, _ = mimetypes.guess_type(filename)
if mimetype and mimetype.startswith('image/'):
image_data = _read_file_or_404(filename, True)
return _render_image(image_data, mimetype)
render_text = _read_file_or_404(filename)
else:
filename = in_filename
if text is not None:
render_text = (text.read() if hasattr(text, 'read')
else str(text))
else:
render_text = _read_file_or_404(filename)
favicon = assets.get('favicon', None)
return _render_page(render_text, filename, gfm, context,
username, password,
render_offline, render_wide,
style_urls, styles, favicon)
def delete_file(name, filename):
wh = get_warehouse()
parcel = get_or_404(wh.get_parcel, name, _exc=KeyError)
if not authorize_for_parcel(parcel):
flask.abort(403)
if not parcel.uploading:
flask.abort(403)
if flask.request.method == 'POST':
filename = secure_filename(filename)
file_path = flask.safe_join(parcel.get_path(), filename)
wh.logger.info("Delete file %r for parcel %r (user %s)",
filename, parcel.name, _current_user())
try:
os.unlink(file_path)
parcel_file_deleted.send(parcel)
flask.flash("File %s was deleted." % name, 'system')
except OSError:
flask.flash("File %s was not deleted." % name, 'system')
return flask.redirect(flask.url_for('parcel.view', name=name))
else:
return flask.render_template('parcel_file_delete.html',
parcel=parcel,
filename=filename)
def files_delete(path):
"""
unlink(path) -> DELETE /files/<path>?op=unlink
200
404 File not found.
rmdir(path) -> DELETE /files/<path>?op=rmdir
200
404 File not found.
"""
if 'op' not in flask.request.args:
return 'Missing operation.', 400
op = flask.request.args['op']
path = flask.safe_join(app.config['serve_dir'], path)
if op == 'unlink':
try:
os.unlink(path)
except FileNotFoundError:
return 'File not found.', 404
return ""
elif op == 'rmdir':
try:
os.rmdir(path)
except FileNotFoundError:
return 'File not found.', 404
except OSError as e:
return 'Errno {}'.format(e.errno), 400
return ""
return 'Unknown operation.', 400
def files_put(path):
"""
write(path, data, offset) -> PUT /files/<path>
{"data": "<base64 str>", "offset": <offset int>}
200 <bytes written int>
404 File not found.
"""
path = flask.safe_join(app.config['serve_dir'], path)
payload = flask.request.json
if 'data' not in payload:
return 'Missing data.', 400
elif 'offset' not in payload:
return 'Missing offset.', 400
data = base64.b64decode(payload['data'])
offset = int(payload['offset'])
# Open, seek, write, close
try:
fd = os.open(path, os.O_WRONLY)
os.lseek(fd, offset, os.SEEK_SET)
n = os.write(fd, data)
os.close(fd)
except FileNotFoundError:
return 'File not found.', 404
return flask.jsonify({"count": n})
def files_post(path):
"""
create(path) -> POST /files/<path>?op=create
200
mkdir(path) -> POST /files/<path>?op=mkdir
200
400 Directory exists.
"""
if 'op' not in flask.request.args:
return 'Missing operation.', 400
op = flask.request.args['op']
path = flask.safe_join(app.config['serve_dir'], path)
if op == "create":
fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0o755)
os.close(fd)
return ""
elif op == "mkdir":
try:
fd = os.mkdir(path, 0o755)
except FileExistsError:
return 'Directory exists.', 400
return ""
return 'Unknown operation.', 400
def write():
""" Create new page with <page_name> filename in <PAGES_FOLDER> const. """
if request.method == 'POST':
page_name = escape(request.form.get('title', None))
content = request.form.get('content', None)
create = request.form.get('create', '0') # default zero; TODO: rewrite this
if not page_name or page_name == '.gitignore' or '.' in page_name or '/' in page_name:
flash('Enter correct title', 'error')
return redirect(url_for('write'))
page_file = safe_join(app.config['PAGES_FOLDER'], page_name)
if create == '1' and os.path.isfile(page_file):
flash('Page already exist with same name!', 'error')
else:
try:
if create != '1':
dump_page(page_name)
with open(page_file, 'w') as f:
f.write(content)
settings_write(page_name, 'last_author', session['username'])
settings_write(page_name, 'last_change', int(time.time()))
flash('Success!', 'info')
return redirect(url_for('page', page_name=page_name))
except OSError:
flash('Can not save page!', 'error')
except Exception:
abort(500)
return render_template('editor.html', context={})
def get_meta(db_id=None):
with shared_lock():
path = safe_join(WRK_DB_DIR, db_id.lower())
path = os.path.join(path, META_FILE_NAME)
if not os.path.exists(path) or not os.path.isfile(path):
raise NotFound()
with gzip.open(path, 'rt') as f:
data = f.read()
data = json.loads(data)
return jsonify(data)
def favicon():
filename = "favicon.ico"
cache_timeout = current_app.get_send_file_max_age(filename)
favicon_path = safe_join(current_app.static_folder, "favicons")
return send_from_directory(
favicon_path,
filename,
mimetype="image/vnd.microsoft.icon",
cache_timeout=cache_timeout,
)
def raw(path):
if not is_staff("cs61a"):
abort(403)
path = safe_join(get_working_directory(get_host_username()), "published",
path)
try:
return send_file(path, cache_timeout=-1)
except FileNotFoundError:
return "", 404
def view_pdf():
if request.method == 'POST':
try:
filename = request.get_json()['filename']
pdf_path = Config.PDF_PATH
safe_path = safe_join(pdf_path, filename)
return send_file(safe_path, as_attachment=False)
except FileNotFoundError:
return "File not found"
except Exception as exp:
return str(exp)
else:
try:
safe_path = safe_join(Config.PDF_PATH, 'pdf-test.pdf')
return send_file(safe_path, as_attachment=False)
except FileNotFoundError:
return "File not found"
except Exception as exp:
return str(exp)
def store_blob(self, agent, blobid, data):
"""Store a blob"""
try:
fdir = Path(config.AGENTBLOB_PATH) / agent.uuid
fdir.mkdir(mode=0o750, exist_ok=True)
fpath = Path(safe_join(str(fdir), blobid))
fpath.write_bytes(data)
return True
except:
return False
def _download_path(self, project):
container = self._container(project)
if isinstance(uploader, local.LocalUploader):
filepath = safe_join(uploader.upload_folder, container)
else:
print(
"The method Exporter _download_path should not be used for Rackspace etc.!"
) # TODO: Log this stuff
filepath = container
return filepath
def remove_file(response):
try:
os.remove(safe_join(AGENTS_PATH, DOWNLOADS_DIR, path))
except NotFound as e:
return flask.make_response(
flask.jsonify({
MESSAGE_KEY:
'The requested URL was not found on the server'
}), HTTPStatus.NOT_FOUND)
return response
def del_req_codefile(path, did):
_, fpath = build_path(path, did)
fname = safe_join(path, did)
try:
os.remove(path)
return jsonify({'status': 'ok', 'result': True})
except IOError:
raise Exception404("I can't remove %s" % fname)
except:
raise Exception400("An error happened")
def router(path: str):
"""
Custom router function that can differ between model pages and their parent directories (for which it needs to
generate index pages).
:param model:
:return:
"""
# support annoying double-slash URLs, redirecting to the real page
original_path = str(path)
while "//" in path:
path = path.replace("//", "/")
if path != original_path:
return redirect(url_for("views.router", path=path))
# resolve path inside models dir
path = path.replace("/index.html", "")
request_dir = safe_join(models_dir(), path)
# if the directory doesn't exist, we emit a 404
if not os.path.isdir(request_dir):
raise NotFound("Could not find models dir")
# let's see if this is an index page or a "content" dir
# the detection works by checking for subdirectories; if there are any, we generate an index page, otherwise we
# try to generate a model page
dir_contents = os.listdir(request_dir)
# completely empty directories should not be visible, similar like how Git works
if not dir_contents:
raise NotFound()
has_subdirs = any(
[os.path.isdir(safe_join(request_dir, i)) for i in dir_contents])
if has_subdirs:
return render_index_page(path)
else:
return render_model_page(path)
def audio(audio_file_key):
"""
Return audio from audio file URL in `audio_file_key`
Parameters
----------
audio_file_key: str
The encrypted key that contains a dictionary that included an item keyed by 'path' which is the location of the
audio file
Returns
-------
flask.Response
"""
if app.config['AUDIO_CODEC'] == 'wav':
file_format = '.wav'
elif app.config['AUDIO_CODEC'] == 'mp3':
file_format = '.mp3'
if app.config['ENCRYPT_AUDIO_STIMULI_URLS']:
try:
audio_file_dict = utilities.decrypt_data(str(audio_file_key))
# can also assert that this file is for this specific participant and condition
assert (audio_file_dict['p_id'] == session['participant_id'])
assert (audio_file_dict['g_id'] in session['condition_group_ids'])
filename = audio_file_dict['URL']
except (ValueError, TypeError):
filename = audio_file_key + file_format
else:
filename = audio_file_key + file_format
if app.config['EXTERNAL_FILE_HOST']:
# return send_file_partial(app.config['AUDIO_FILE_DIRECTORY']+filename)
return send_file_partial_hack(
safe_join(app.config['AUDIO_FILE_DIRECTORY'], filename))
else:
return send_file_partial(
safe_join(
safe_join(app.root_path, app.config['AUDIO_FILE_DIRECTORY']),
filename))
def get_size(path, skip_dirs):
size = 0
for root, dirs, files in os.walk(path):
size += sum(
os.path.getsize(safe_join(root, name)) for name in files)
for skip_dir in skip_dirs:
if skip_dir in dirs:
dirs.remove(skip_dir)
return size
def read_environment_from_disk(env_directory,
project_uuid) -> Optional[Environment]:
try:
with open(safe_join(env_directory, "properties.json"), "r") as file:
env_dat = json.load(file)
with open(safe_join(env_directory, _config.ENV_SETUP_SCRIPT_FILE_NAME),
"r") as file:
setup_script = file.read()
e = Environment(**env_dat)
e.project_uuid = project_uuid
e.setup_script = setup_script
return e
except Exception as e:
current_app.logger.error(
"Could not get environment from env_directory %s. Error: %s" %
(env_directory, e))
def serialize_environment_to_disk(environment, env_directory):
environment_schema = EnvironmentSchema()
# treat setup_script separately
with open(safe_join(env_directory, "properties.json"), "w") as file:
environmentDICT = environment_schema.dump(environment)
# don't serialize project_uuid
del environmentDICT["project_uuid"]
# setup scripts is serialized separately
del environmentDICT["setup_script"]
file.write(json.dumps(environmentDICT))
# write setup_script
with open(safe_join(env_directory, _config.ENV_SETUP_SCRIPT_FILE_NAME),
"w") as file:
file.write(environment.setup_script)
def extract_data(message, root, min_size=0.0, max_size=1.0):
"""Replaces large Data values with pointers to offloaded data.
Git LFS (https://git-lfs.github.com/) is convenient because it lives in the
same repo as the associated Reaction records. However, it is not possible to
get a permanent URL for the uploaded data because it is only committed to
the PR branch. We have (at least) these options:
1. Modify the URL just before or after the PR is merged to point to the
correct branch.
2. Modify the URL to point to its eventual destination (in the `main`
branch) and deal with broken links during submission review, or
3. Use relative paths (relative to the repository root). This means that
users will have to traverse the repo manually to access referenced
data instead of simply following a URL.
4. Merge the data immediately in another repo so the URL is permanent.
I think (2) is the best option because it yields URLs that will eventually
work and it is simpler than (1). I don't like option (4) because it requires
data to be committed and merged before review.
Args:
message: Protocol buffer message.
root: Text root of the repository.
min_size: Float minimum size of data before it will be written (in MB).
max_size: Float maximum size of data to write (in MB).
Returns:
Set of text filenames; the generated Data files.
"""
dirname = tempfile.mkdtemp()
data_messages = message_helpers.find_submessages(message,
reaction_pb2.Data)
filenames = set()
for data_message in data_messages:
data_filename, data_size = write_data(data_message,
dirname,
min_size=min_size,
max_size=max_size)
if data_filename:
basename = os.path.basename(data_filename)
output_filename = message_helpers.id_filename(basename)
with_root = flask.safe_join(root, output_filename)
if os.path.exists(with_root):
warnings.warn(f'Target Data blob already exists: {with_root}')
else:
os.makedirs(os.path.dirname(with_root), exist_ok=True)
shutil.copy2(data_filename, with_root)
filenames.add(with_root)
data_message.url = urllib.parse.urljoin(DATA_URL_PREFIX,
output_filename)
logging.info('Created Data link (%g MB): %s', data_size, with_root)
shutil.rmtree(dirname)
return filenames
def main():
""" Main route, entry point for react. """
validate_auth()
## issue with path resolution after build
return send_from_directory(
#todo: remove templates directory reference; index.html isn't a jinja template
safe_join(
current_app.config.get("STATIC_DIR") or current_app.static_folder,
'templates'),
'index.html',
cache_timeout=-1)
def put_file(loc):
token = flask.request.args.get('token')
sess_dir = make_dir_name(token)
if not os.path.isdir(sess_dir):
os.mkdir(sess_dir)
file_loc = flask.safe_join(sess_dir, loc)
with open(file_loc, 'wb') as f:
f.write(flask.request.data)
return flask.jsonify({'disk': file_loc})
def get_3d(session, folders, filename):
folders = folders.split('|')
path = safe_join(prefix, session, *folders)
path = safe_join(path, 'pose-3d', filename + '.csv')
data = pd.read_csv(path)
cols = [x for x in data.columns if '_error' in x]
bodyparts = [c.replace('_error', '') for c in cols]
vecs = []
for bp in bodyparts:
vec = np.array(data[[bp+'_x', bp+'_y', bp+'_z']])
vecs.append(vec)
vecs = np.array(vecs).swapaxes(0, 1)
lengths = np.linalg.norm(vecs[:,0] - vecs[:,1], axis=1)
L = np.median(lengths)
vecs = vecs / L * 0.22;
return jsonify(vecs.tolist())
def flask_server_tiles(filename):
""" Serve up a file from the tile server location """
global map_settings
if map_settings['tile_server_enabled']:
_filename = flask.safe_join(map_settings['tile_server_path'], filename)
if os.path.isfile(_filename):
return flask.send_file(_filename)
else:
flask.abort(404)
else:
flask.abort(404)
def edit_content(course, content_path, TOC: TableOfContent):
try:
content = TOC.get_page_from_path("%s.rst" % content_path)
except ContentNotFoundError:
content = TOC.get_content_from_path(content_path)
if request.method == "POST":
inpt = request.form
if "new_content" not in inpt:
return seeother(request.path)
else:
if type(content) is Chapter:
with open(safe_join(syllabus.get_pages_path(course), content.path, "chapter_introduction.rst"), "w") as f:
f.write(inpt["new_content"])
else:
with open(safe_join(syllabus.get_pages_path(course), content.path), "w") as f:
f.write(inpt["new_content"])
return seeother(request.path)
elif request.method == "GET":
return render_template("edit_page.html", course=course, content_data=get_content_data(course, content),
content=content, TOC=TOC, cheat_sheet=get_cheat_sheet(), enable_preview=syllabus.get_config().get("enable_editing_preview", False))
def get_return_path(project, fpath, root_upload_path=None):
"""Splice upload_path and project from fpath and return the path
shown to the user and POSTed to Metax.
"""
if not root_upload_path:
root_upload_path = current_app.config.get("UPLOAD_PATH")
base_path = safe_join(root_upload_path, project)
ret_path = os.path.normpath(fpath[len(base_path):])
return ret_path if ret_path != "." else "/"
def url_handle():
url = request.args.get('url', '')
if not url or url == '':
return redirect(url_for('index'))
if "auth=1" in url.split("&"):
abort(401)
if '://' not in url:
url = "http://" + url
if "readsy.co" in url:
return render_template('spritz.html', text="Try parsing a different website!")
ext = url.split('.')[-1].lower()
if ext in ALLOWED_EXTENSIONS:
try:
r = requests.get(url, stream=True)
except:
abort(400)
if 'content-length' in r.headers:
if int(r.headers['content-length']) > application.config['MAX_CONTENT_LENGTH']:
abort(400)
if r.status_code == 200:
fullname = url.split('/')[-1]
filename = fullname.split('.')[0]
path = safe_join(application.config['UPLOAD_FOLDER'], fullname)
with open(path, 'wb') as f:
for chunk in r.iter_content():
f.write(chunk)
if ext == "pdf":
s = PDFhelper(path)
return render_template('spritz.html', text=s, filename=fullname, titleText=fullname)
elif ext == "txt":
s = TXThelper(path)
return render_template('spritz.html', text=s, filename=fullname, titleText=fullname)
else:
abort(400)
return
try:
g = Goose()
article = g.extract(url=url)
s = cleantext(article.cleaned_text)
# if article.top_image:
# img = article.top_image
return render_template('spritz.html', text=s, filename=url.split('//')[1], titleText=article.title)
except:
abort(400)
def get_containing_chapters_of(self, content):
str_containing_chapters = content.path.split(os.sep)[:-1]
result = []
actual_path = []
for chapter_str in str_containing_chapters:
actual_path.append(chapter_str)
joined_path = safe_join(*actual_path)
result.append(
Chapter(joined_path, self.path_to_title_dict[joined_path],
self.toc_path))
return result
def delete(path):
full_path = Path(safe_join(app.config["UPLOAD_FOLDER"], path))
if os.path.exists(full_path):
try:
os.remove(full_path)
return json.dumps(str(path)), 200
except OSError as ex:
return json.dumps("Error while deleting file {}:\n{}".format(
path, ex)), 500
else:
return json.dumps("File {} not found".format(path)), 404
def test_safe_join(self):
# Valid combinations of *args and expected joined paths.
passing = (
(('a/b/c', ), 'a/b/c'),
((
'/',
'a/',
'b/',
'c/',
), '/a/b/c'),
((
'a',
'b',
'c',
), 'a/b/c'),
((
'/a',
'b/c',
), '/a/b/c'),
(
('a/b', 'X/../c'),
'a/b/c',
),
(
('/a/b', 'c/X/..'),
'/a/b/c',
),
# If last path is '' add a slash
(
(
'/a/b/c',
'',
),
'/a/b/c/',
),
# Preserve dot slash
(
(
'/a/b/c',
'./',
),
'/a/b/c/.',
),
(
('a/b/c', 'X/..'),
'a/b/c/.',
),
# Base directory is always considered safe
(('../', 'a/b/c'), '../a/b/c'),
(('/..', ), '/..'),
)
for args, expected in passing:
assert flask.safe_join(*args) == expected
def post_inginious(course):
inpt = request.form
data = parse.urlencode(inpt).encode()
inginious_config = syllabus.get_config()['courses'][course]['inginious']
path = safe_join(inginious_config.get("simple_grader_pattern", "/"), inginious_config['course_id'])
inginious_sandbox_url = urllib.parse.urljoin(inginious_config["url"], path)
req = urllib_request.Request(inginious_sandbox_url, data=data)
resp = urllib_request.urlopen(req)
response = make_response(resp.read().decode())
response.headers['Content-Type'] = 'text/json'
return response
def _upload_path(self):
container = self._container()
filepath = None
if isinstance(uploader, local.LocalUploader):
filepath = safe_join(uploader.upload_folder, container)
if not os.path.isdir(filepath):
os.makedirs(filepath)
return filepath
current_app.logger.error('Failed to generate upload path {0}'.format(filepath))
raise IOError('Local Upload folder is missing: {0}'.format(filepath))
def urlget(urlshort):
fp = safe_join('files', urlshort)
if not isfile(fp):
abort(404)
r = BytesIO()
mime = magic.from_file(fp, mime=True)
with open(fp, 'rb') as fo:
r.write(fo.read())
r.seek(0)
remove(fp)
return send_file(r, mimetype=mime)
def get_user_upload(subdirectory):
"""Return the location of a subdirectory for uploads. This also uses a
subdirectories path component in the main user upload directory.
This function is safe to path injection (such as .. in filename).
This function will also care about directory creation if it doesn't exist
yet.
Path example : /upload-dir/subdirectory
"""
user_dir = safe_join(current_app.config["UPLOAD_DIR"], subdirectory)
os.makedirs(user_dir, exist_ok=True)
return user_dir
def upload():
results = []
for filename, file in request.files.items():
if not filename:
continue
file_path = filename.lstrip(os.sep)
target = Path(safe_join(app.config["UPLOAD_FOLDER"], file_path))
target.parent.mkdir(parents=True, exist_ok=True)
file.save(str(target))
results.append(file_path)
return (json.dumps(results), 200)
def download_Annotations(target, database, ID):
print(target, database, ID)
entry = search_for_download(database, ID).next()
annot = open(
os.path.join(app.config['ANNOTATION_FILES'], 'GOannotation_Ago20.txt'),
'r').readlines()
del annot[0]
annot = dict(map(lambda x: (x.split('\t')[0], x.split('\t')), annot))
header = [
'Dataset', 'neXtProt_ID', 'EnsembleID', 'gene_name', 'uPE1',
'PE_level', 'Chr', 'DB', 'Ontology', 'P.value', 'Description', '\n'
] if entry['UPE'] else [
'Dataset', 'neXtProt_ID', 'EnsembleID', 'gene_name', 'uPE1',
'PE_level', 'Chr', 'DB', 'Ontology', 'Description', '\n'
]
SEP = '\t'
file_name = target.replace(
' ', '_').title() + 'Annot_' + time.strftime("%Y%m%d-%H%M") + '.tsv'
with open(os.path.join('tmp', file_name), 'w') as outfl:
outfl.write(SEP.join(header))
anchor = SEP.join([
entry['dataset'], entry['nexprot_id'], entry['_id'],
entry['gene_name'],
str(entry['UPE']), entry['PE_level'],
str(entry['chr'])
])
for go in entry['GO_con']:
go_id = go['id']
if entry['UPE']:
go_pval = go['pval']
line = anchor + SEP + 'Gene Ontology' + SEP + go_id + SEP + go_pval + SEP + annot[
go_id][2] + '\n'
else:
line = anchor + SEP + 'Gene Ontology' + SEP + go_id + SEP + annot[
go_id][2] + '\n'
outfl.write(line)
for msig in entry['Msig_con']:
msig_id = msig['id']
if entry['UPE']:
msig_pval = msig['pval']
line = anchor + SEP + 'MSigDB' + SEP + msig_id + SEP + msig_pval + '\n'
else:
line = anchor + SEP + 'MSigDB' + SEP + msig_id + '\n'
outfl.write(line)
for mala in entry['Mala_con']:
mala_id = mala['id']
if entry['UPE']:
mala_pval = mala['pval']
line = anchor + SEP + 'Disease' + SEP + mala_id + SEP + mala_pval + '\n'
else:
line = anchor + SEP + 'Disease' + SEP + mala_id + '\n'
outfl.write(line)
safe_path = safe_join(app.config["CLIENT_FILES"], file_name)
return send_file(safe_path, as_attachment=True)
