def get_data(query, lang):
if lang != 'en':
return None
url = f'https://www.macmillandictionary.com/dictionary/british/{query}'
headers = {
'User-Agent': 'script',
'Accept': 'text/html',
}
resp = requests.get(url, headers=headers)
resp.raise_for_status()
soup = BeautifulSoup(resp.text, 'html.parser')
#get transcription
transcriptions = soup.find_all(class_='PRON')
for t in transcriptions:
yield ('transcription',
Term(text=stripped_text(t).replace('/', ''),
lang=lang,
region=None))
#get tags
crop_text = stripped_text(soup.find(class_='zwsp'))
#remove bad substring
part_speech = stripped_text(soup.find(class_='PART-OF-SPEECH')).replace(
crop_text, '')
syntax_coding = stripped_text(soup.find(class_='SYNTAX-CODING'))
yield ('tag', Term(text=part_speech, lang=lang, region=None))
yield ('tag', Term(text=syntax_coding, lang=lang, region=None))
#get defenition
defenitions = soup.find_all(class_='DEFINITION')
for d in defenitions:
yield ('definition', Term(text=stripped_text(d),
lang=lang,
region=None))
#get examples
examples = soup.find_all(class_='EXAMPLES')
for e in examples:
yield ('in', Term(text=stripped_text(e), lang=lang, region=None))
examples = soup.find_all(class_='PHR-XREF')
for e in examples:
yield ('in', Term(text=stripped_text(e), lang=lang, region=None))
#get synonyms
synonyms = soup.find_all(class_='synonyms')
for allsyn in synonyms:
subsynonyms = allsyn.find_all(class_='theslink')
for syn in subsynonyms:
if (not '...' in syn.text):
yield ('synonym',
Term(text=stripped_text(syn), lang=lang, region=None))
#get audio
audio = soup.find(class_='audio_play_button')
yield ('audio', File(url=audio['data-src-mp3'], region=None))
yield ('audio', File(url=audio['data-src-ogg'], region=None))
def get(self):
file_id = self.request.get("id")
file = self.get_file(file_id)
if file is None:
file = File()
else:
if self.get_file_permission(file) < base.ACCESS_READ:
file = File()
template_values = {'file': file}
path = self.get_template_path('fileinfo.html')
self.response.out.write(template.render(path, template_values))
def upload_file(path):
if request.method == "POST":
file = request.files['file']
filename = ".".join(secure_filename(
file.filename).split(".")[:-1]) + "_" + str(current_user.id)
ext = os.path.splitext(file.filename)[-1]
filename = filename + ext
file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
file.save(file_path)
parent_path = request.form.get('parent_path')
size = os.path.getsize(file_path)
filedata = File(user_id=current_user.id,
child_path=filename,
type=ext,
original_path="/home/ubuntu/data",
filename=filename,
size=size,
parent_path='/' + parent_path.strip('/') + '/')
db.session.add(filedata)
db.session.commit()
return redirect(url_for("data", path=path))
else:
return render_template("upload_file.html", path=path)
def upload(self):
celery = create_celery(current_app)
if request.method == 'POST':
if 'folder' in request.files:
files = request.files.getlist('folder')
for f in files:
filename = secure_filename(f.filename)[3:]
symbol, short_name, contract_date = _parse_contract_date(
filename)
s3.put_object(Bucket=os.environ.get('S3_BUCKET_NAME'),
Key='downloads2/{}'.format(f.filename),
Body=f)
url = 'https://s3.amazonaws.com/{}/downloads2/{}/{}'.format(
os.environ.get('S3_BUCKET_NAME'), symbol, short_name)
db.session.add(
File(filename=filename,
path=url,
symbol=symbol,
contract_date=contract_date))
db.session.commit()
# celery.send_task('tasks.write_filetable', args=(filename, url, symbol, contract_date))
flash("Your files have been successfully uploaded!", "success")
return redirect(request.url)
return redirect(request.url)
return self.render('admin/upload_file.html')
def post(self):
user = users.get_current_user()
if user == None:
self.redirect('/')
else:
current_user = MyUser.get(user.user_id())
dir_id = self.request.get('dir')
owner_user, directory, parent = check(current_user.key, dir_id)
if owner_user and directory and (
directory.is_users_dir(current_user.key)
or has_write_permission(current_user.key, parent)):
for upload in self.get_uploads():
blobinfo = blobstore.BlobInfo(upload.key())
my_file = File(name=blobinfo.filename, blob=upload.key())
if not directory.file_exists(my_file.name):
directory.files.append(my_file)
directory.put()
self.redirect(self.request.referer)
def file_factory(**kwargs):
base_parameters = dict(bucket_path='/test-job/my-foto.png',
created_at=datetime.datetime(2019, 3, 1))
file = File(**dict(**kwargs, **base_parameters))
db.session.add(file)
db.session.commit()
return file
def add_file_to_database(submitter, title, original_name, description,
file_content, file_mime, file_hash):
"""
This:
1) Writes a file to the disk ({UPLOAD_FOLDER}/hash),
2) Attempts to generate a transcript of the file,
3) Saves the file to the database.
"""
# save file to UPLOAD_FOLDER/file_hash
save_location = ''.join(
[current_app.config['UPLOAD_FOLDER'], "/", file_hash])
with open(save_location, "wb+") as storage:
storage.write(file_content) # actually write the file
try:
text = textract.process(
save_location,
language='eng',
extension=get_extension_from_mimetype(file_mime)).decode('utf-8')
except (textract.exceptions.UnknownMethod, textract.exceptions.ShellError):
text = ""
new_file = File(title=title,
original_name=original_name,
description=description,
file_path=save_location,
file_mime=file_mime,
transcript=text,
file_hash=file_hash,
submitter=submitter)
APP_DATABASE.session.add(new_file) # add to database
APP_DATABASE.session.commit()
return new_file.id
def train():
net_id = int(request.form["net_id"])
net = net_logic.get(net_id) # type: Net
separator = request.form["separator"]
text_column = int(request.form["text_column"])
tag_column = int(request.form["tag_column"])
positive_tag = request.form["positive_tag"]
negative_tag = request.form["negative_tag"]
uploaded_files = []
for filename, file_stream in request.files.items():
file = File()
file.separator = separator
file.text_column = text_column
file.tag_column = tag_column
file.positive_tag = positive_tag
file.negative_tag = negative_tag
uploaded_files.append(file_logic.save(file, file_stream))
extend_vocabulary = True if request.form[
"extend_vocabulary"] == "true" else False
net = net_logic.init_training(net.id, [file.id for file in uploaded_files],
extend_vocabulary)
net_info = net.to_dict(_hide=[])
return jsonify(net_info)
def setUp(self):
self.group, created = Group.objects.get_or_create(name='test')
self.user, created = DjangoUser.objects.get_or_create(username=self.username)
self.user.set_password(self.password)
self.user.save()
perspective, created = Perspective.objects.get_or_create(
name='default')
perspective.set_default_user()
perspective.save()
ModuleSetting.set('default_perspective', perspective.id)
self.folder = Folder(name='test')
self.folder.set_default_user()
self.folder.save()
self.document = Document(title='test_document', folder=self.folder)
self.document.set_default_user()
self.document.save()
self.file = File(name='test_file', folder=self.folder)
self.file.set_default_user()
self.file.save()
self.link = WebLink(title='test', folder=self.folder, url='test')
self.link.set_default_user()
self.link.save()
def get_data(text, lang='en'):
if lang != 'en':
return None
url = f'https://howjsay.com/mp3/{quote(text)}.mp3'
if utils.url_exists(url):
yield ('audio', File(url=url, region=None))
def edit_person(slug):
try:
p = Person.objects.get(site=g.site.domain, slug=slug)
except Person.DoesNotExist:
abort(404)
if not g.site.domain == g.user:
abort(403)
if request.method == "POST":
reqfile = request.files.get('file')
if reqfile:
f = File()
f.site = g.site.domain
f.name = reqfile.filename
f.content_type = reqfile.mimetype
f.slug, f.content_length = save_file(
reqfile, request.config["UPLOAD_FOLDER"])
f.save()
p.name = request.form.get("name")
p.title = request.form.get("title")
p.phone = request.form.get("phone")
p.email = request.form.get("email")
p.twitter = request.form.get("twitter")
p.description = request.form.get("description")
if reqfile:
p.image_slug = f.slug
p.save()
return redirect(url_for(".show_person", slug=p.slug))
return render_template("edit_person.html", person=p)
def train():
model_id = int(request.form["model_id"])
model = db.session.query(EmbeddingModel).get(
model_id) # type: EmbeddingModel
if model is None:
raise ValueError("No embedding model found by id {0}".format(model_id))
separator = request.form["separator"]
text_column = int(request.form["text_column"])
tag_column = int(request.form["tag_column"])
positive_tag = request.form["positive_tag"]
negative_tag = request.form["negative_tag"]
uploaded_files = []
for filename, file_stream in request.files.items():
file = File()
file.separator = separator
file.text_column = text_column
file.tag_column = tag_column
file.positive_tag = positive_tag
file.negative_tag = negative_tag
uploaded_files.append(file_logic.save(file, file_stream))
extend_vocabulary = True if request.form[
"extend_vocabulary"] == "true" else False
model = emb_model_logic.init_training(model_id,
[file.id for file in uploaded_files],
extend_vocabulary)
return jsonify(model.to_dict())
def edit_post(year, month, day, slug):
try:
p = Post.objects.get(site=g.site.domain,
year=year,
month=month,
day=day,
slug=slug)
except Post.DoesNotExist:
abort(404)
if not g.site.domain == g.user:
abort(403)
if request.method == "POST":
reqfile = request.files.get('file')
if reqfile:
f = File()
f.site = g.site.domain
f.name = reqfile.filename
f.content_type = reqfile.mimetype
f.slug, f.content_length = save_file(
reqfile, current_app.config["UPLOAD_FOLDER"])
f.save()
p.name = request.form.get("name")
p.text = request.form.get("text")
if reqfile:
p.image_slug = f.slug
p.save()
return redirect(
url_for("post", year=p.year, month=p.month, day=p.day,
slug=p.slug))
return render_template("edit_post.html", post=p)
def init_files(self, pathname):
_files = [pathname]
file_count = 0
line_count = 0
while _files:
pathname = _files.pop(0)
if self.files.isdir(pathname):
dirs, files = self.files.getdir(pathname)
for i in dirs + files:
_files.append(os.path.join(pathname, i))
else:
f = File(pathname)
cnt = self.files.getlinecount(pathname)
f.filetype = self.files.gettype(pathname)
f.linecount = cnt
self.session.add(f)
file_count += 1
line_count += cnt
self.pathname_to_obj[pathname] = f
self.session.commit()
self.track_info['file_count'] = file_count
self.track_info['line_count'] = line_count
def is_project_file(filename, project_id):
"""
Checks, if a file belongs to a project. Checks both project folder
and database.
:param filename: a filepath.
:type filename: String
:param project_id: Project id from database.
:type project_id: Integer
:rtype: Boolean
"""
filebase = os.path.basename(filename)
fileabs = os.path.abspath(filename)
if File.get('exists', File.path == fileabs, File.project_id == project_id):
return True
project = Project.get_by_id(project_id)
file_project_path = filesystem.search_file(filebase, project.dir)
project = None
if file_project_path:
try:
File(file_project_path, project_id)
return True
except (ItemAlreadyExistsException, SQLAlchemyError):
pass
return False
def RegisterFile(Service=None, FileName=None, ProvisionedSpace="10G"):
if Service is None:
raise StorageError('RegisterFile(): Service can not be None')
if FileName is None:
raise StorageError('RegisterFile(): FileName can not be None')
vfilespace = StringSizeToBytes(ProvisionedSpace)
if Service.freespace - vfilespace > 0:
NewFile = File()
NewFile.vfilename = FileName
NewFile.ufid = GetUniqueFileID(FileName)
NewFile.pfilesize = 0
NewFile.vfilesize = vfilespace
NewFile.service = Service
NewFile.pfilename = GetPhysicalFileName(Service.localpath, FileName)
NewFile.status = 'O'
NewFile.save()
SFreeSpace = CalculateFreeSpace(Service)
Service.freespace = SFreeSpace
Service.save()
return NewFile
else:
raise StorageError('RegisterFile(): No have left space')
def upload_flash(request):
"""
Helper function that processes the file upload. Used by flash player.
:Return: HttpResponse of 0 or 1
"""
try:
if (request.method == 'POST' and 'file_upload' in request.FILES
and 'slug' in request.GET):
upload = File()
upload.file_upload = request.FILES['file_upload']
upload.submission = Submission.objects.get(
slug=request.GET['slug'])
upload.save()
return HttpResponse(1)
else:
return HttpResponse(0)
except StopUpload:
raise
except:
return HttpResponse(0)
def create_instance(self, body_data):
"""
重载修改后:
*.将password字段加密
:param body_data: 创建对象时的传入数据
:return: instance: 创建操作后的对象
"""
password = generate_password_hash(body_data["password"],
method="pbkdf2:sha256")
body_data["password"] = password
body_data["creator_id"] = g.login_user.id
if "img_url" not in body_data:
body_data[
"img_url"] = "https://gw.alipayobjects.com/zos/rmsportal/BiazfanxmamNRoxxVxka.png"
# 判断该模型是否支持所有输入属性
for item in body_data:
if not hasattr(self.model, item):
del body_data[item]
# 创建对象
user = self.model(**body_data)
db.session.add(user)
db.session.commit()
# 创建空间
space_data = {
"name": user.username + "'s private space",
"space_type": "private",
"own_user_id": user.id
}
space = Space(**space_data)
db.session.add(space)
db.session.commit()
# 创建桶
bucket_data = {
"name": space.name + "'s 1st bucket",
"space_id": space.id
}
bucket = Bucket(**bucket_data)
db.session.add(bucket)
db.session.commit()
# 创建空间根目录
bucket_data = {
"object_name": space.name + "'s root",
"object_type": "folder",
"object_size": 0,
"creator_id": user.id,
"bucket_id": bucket.id
}
folder_root = File(**bucket_data)
db.session.add(folder_root)
db.session.commit()
# 关联空间与根目录
space.root_folder = folder_root
db.session.commit()
return user
def upload():
if not request.files['file'].filename:
flash(u'请上传文件')
return redirect(url_for('index'))
f = request.files['file']
files = [file.name for file in File.query.all()]
if f.filename in files:
flash(u'文件已存在, 直接下载')
return redirect(url_for('index'))
# 将文件存至服务器
dirname = os.path.join(os.path.dirname(__file__), 'files')
try:
f.save(os.path.join(dirname, f.filename))
except Exception as e:
logger.error(e)
return redirect(url_for('index'))
# 将信息存入数据库
file = File(name=f.filename, size=request.content_length)
try:
db.session.add(file)
db.session.commit()
except Exception as e:
logger.error(e)
return redirect(url_for('index'))
return redirect(url_for('index'))
def GetOutput(self):
# if not self.check_user():
# return
file_id = self.request.get("id")
file = self.get_file(file_id)
if file is None:
file = File()
file.put()
else:
if self.get_file_permission(file) < base.ACCESS_READ:
self.redirect('/')
return
head = file.head
if head is None:
file_text = "Welcome to ZenTxt!"
revisions = []
else:
#file_text = cgi.escape(head.content)
file_text = head.content
revisions = self.get_revisions(file)
template_values = {
'user': self.get_current_user(),
'file_id': file_id,
'revisions': revisions,
'file_text': file_text,
'login_url': users.create_login_url(self.request.uri)
}
path = self.get_template_path('file.html')
return template.render(path, template_values)
def add_files(request, nonce=''):
submission = get_object_or_404(UserSubmission, nonce=nonce)
# don't let you edit the object if it's processing or processed.
# this is CRITICAL, be very careful modifying this stanza or the above!
if submission.is_processing():
return HttpResponseRedirect(reverse(process, kwargs={'nonce': submission.nonce}))
elif not submission.can_process():
return HttpResponseRedirect(reverse(review, kwargs={'nonce': submission.nonce}))
files = File.objects.filter(usersubmission=submission)
fileinstance = File(usersubmission=submission)
if request.method == 'POST':
form = FileForm(request.POST, request.FILES, instance=fileinstance)
if form.is_valid():
uploaded_file_instance = form.save(commit=False)
uploaded_file_instance.filename = request.FILES['path'].name
uploaded_file_instance.save()
return HttpResponseRedirect(submission.get_absolute_url())
else:
form = FileForm(instance = fileinstance)
return render_to_response('bookmaker/add_files.html',
{ 'form': form, 'submission': submission, 'files': files },
context_instance=RequestContext(request))
async def post(self):
with self.make_session() as session:
self._session = session
args, user = await self.init_API_request()
if user:
url = args.get("file_URL")
a = urlparse(url)
filename = os.path.join("uploads", os.path.basename(a.path))
filename, headers = await tornado.ioloop.IOLoop.current(
).run_in_executor(
None,
urllib.request.urlretrieve,
url,
filename,
)
if filename:
file = File(user.id, 0, filename)
session.add(file)
session.commit()
json_response = {"filename": filename, "file_id": file.id}
self.write(json_response)
else:
self.write({"error": "Failed to download file"})
def handle_new_file(user, uploaded, bulletin_id, encryption_key=None, rand=Random.new()):
if encryption_key == '':
encryption_key = None
r, c, ep, iv, encryption_key = create_encryption_params(encryption_key, rand)
file = File(
author=user,
bulletin=Bulletin.objects.get(pk=bulletin_id),
name=uploaded.name,
content=uploaded,
rand=r,
check=c,
encryption_params=ep,
date_created=datetime.now(),
date_modified=datetime.now(),
)
if encryption_key is not None:
with tempfile.NamedTemporaryFile() as temp:
cipher = AES.new(encryption_key, AES.MODE_CFB, iv)
for chunk in uploaded.chunks():
encrypted_chunk = cipher.encrypt(chunk)
temp.write(encrypted_chunk)
temp.seek(0)
file.content = DFile(temp)
file.save()
give_file_access(user, user, file, encryption_key)
else:
file.save()
def get_data(text, lang='ru'):
pat = 'https://ru.forvo.com/word/{0}/#{1}'
url = pat.format(urllib.parse.quote(text), lang)
headers = {
'User-Agent': utils.CHROME_USER_AGENT,
'Accept': 'text/html',
}
resp = requests.get(url, headers=headers)
resp.raise_for_status()
soup = BeautifulSoup(resp.text, 'html.parser')
article = soup.find('article', class_='pronunciations')
if article is None:
return None
ul = article.find('ul', class_="show-all-pronunciations")
if ul is None:
return None
li = ul.find_all('li')
parsed_items = [parse_item(t) for t in li]
items = [
t for t in parsed_items if t is not None and utils.url_exists(t['url'])
]
data = {'audio': []}
for item in items:
data['audio'].append(File(url=item['url'], region=None))
return data
def file_upload():
if request.method == 'POST':
logger.info(request.form)
form = FileUploadForm(CombinedMultiDict((request.files, request.form)))
file = request.files.get('file')
form.hash.data = hasher.hash(file)
logger.info(form.hash.data)
if form.validate():
logger.info('Form is valid')
file.stream.seek(0)
timestamp = int(datetime.datetime.now().timestamp())
filename = str(timestamp) + secure_filename(file.filename)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
logger.info(filename + ' Saved')
db_file = File(name=form.upload_name.data,
cve_number=form.cve_number.data,
full_path=os.path.join(app.config['UPLOAD_FOLDER'],
filename),
description=form.description.data,
file_type=form.upload_type.data,
hash=form.hash.data)
db.session.add(db_file)
db.session.commit()
# logger.info(str(db_file) + ' saved to db')
flash('File Uploaded', 'success')
else:
# logger.info(form.errors)
flash_form_errors(form)
if request.referrer:
return redirect(request.referrer)
else:
return redirect(url_for('index'))
def upload(request, name):
if request.method == 'POST':
if request.FILES is None:
return HttpResponseBadRequest('No file[] found')
file = request.FILES[u'files[]']
wrapped_file = UploadedFile(file)
filename = wrapped_file.name
file_size = wrapped_file.file.size
entry = File()
entry.filename = unicode(filename)
entry.username = unicode(name)
entry.size = file_size
entry.slug = generate_slug(8)
entry.data = file
entry.save()
uploaded = [{
'name': filename,
'size': filesizeformat(file_size),
'slug': entry.slug,
'date': entry.created.date().isoformat()
}]
return HttpResponse(json.dumps({'files': uploaded}))
else:
raise Http404
def upload():
file = request.files['file']
try:
days = float(request.form['days'])
except ValueError:
days = 0
try:
hrs = float(request.form['hours'])
except ValueError:
hrs = 0
if file.filename == '' or (days == 0 and hrs == 0):
flash("Incorrect file data!")
else:
expire_at = datetime.datetime.now()
expire_at += datetime.timedelta(days=days, hours=hrs)
expire_at = expire_at.replace(microsecond=0)
new_file = File(file.filename, file.read(), expire_at,
None if current_user.is_anonymous else current_user.id)
db.session.add(new_file)
db.session.commit()
flash(f"File {file.filename} will be stored until {expire_at}!\n"
f"The file can be accessed at "
f"{url_for('file', file_id=new_file.id, _external=True)}!")
return jsonify(message="OK")
def upload_file(f):
print(f.filename, file=sys.stdout)
filename = secure_filename(f.filename)
fullpath = os.path.join(app.config['UPLOAD_FOLDER'], filename)
f.save(fullpath)
newFile = File(name=filename, path=fullpath)
db_session.add(newFile)
db_session.commit()
def GetFile_FromFolderName(self,folder_name):
folder = Submenu.query.filter_by(name = folder_name).first()
folder_id = folder.file_id
file_list = self.drive.ListFile({'q': folder_id}).GetList()
db.session.query(folder).delete()
db.session.commit()
for i,_file in enumerate(file_list):
new_file = File(name = _file['title'],uri = _file['alternateLink'],file_id = _file['id'],submenu = folder)
db.session.commit()
def upload(username, id):
file = request.files['file']
if file is None:
return make_response('Nie podano pliku', 401)
new_file = File(filename=file.filename,
data=file.read(),
bibliography_id=int(id))
bibliography_storage.upload_file(new_file)
return make_response('Pomyślnie dodano plik', 200)
