def RegisterFile(Service=None, FileName=None, ProvisionedSpace="10G"):
if Service is None:
raise StorageError('RegisterFile(): Service can not be None')
if FileName is None:
raise StorageError('RegisterFile(): FileName can not be None')
vfilespace = StringSizeToBytes(ProvisionedSpace)
if Service.freespace - vfilespace > 0:
NewFile = File()
NewFile.vfilename = FileName
NewFile.ufid = GetUniqueFileID(FileName)
NewFile.pfilesize = 0
NewFile.vfilesize = vfilespace
NewFile.service = Service
NewFile.pfilename = GetPhysicalFileName(Service.localpath, FileName)
NewFile.status = 'O'
NewFile.save()
SFreeSpace = CalculateFreeSpace(Service)
Service.freespace = SFreeSpace
Service.save()
return NewFile
else:
raise StorageError('RegisterFile(): No have left space')
def expose_path(group):
form_keys = ["path", "size", "hash", "hash_function", "modified", "signature"]
for key in form_keys:
if key not in request.form:
raise ValueError("Missing form value %s" % key)
path = request.form["path"]
signature = request.form["signature"]
# Don't include signature
d = {k: request.form[k] for k in form_keys[:-1]}
if not security.check_json_sig(d, app.config["BASEJUMP_KEY"], signature):
raise ValueError("Invalid signature provided.")
key = security.sign_path(path, app.config["SECRET_KEY"])
meta = request.form
with db_session() as s:
f = s.query(File).filter(File.key == key).all()
if f:
raise ValueError("This path is already exposed.")
last_modified = datetime.fromtimestamp(int(meta["modified"]))
f = File(path=path, group=group, key=key, size=meta["size"], checksum=meta["hash"], checksumType=meta["hash_function"], modified=last_modified)
s.add(f)
s.commit()
url_path = f.queue_url()
url = request.url_root + url_path
return jsonify({"queue_url": url})
def file_path(slug):
f = File.all().filter('abs_path = ', slug).get()
if f is None:
f = File.all().filter('slug = ', slug).get()
if f is None:
return u''
return f.get_absolute_url()
def GetOutput(self):
# if not self.check_user():
# return
file_id = self.request.get("id")
file = self.get_file(file_id)
if file is None:
file = File()
file.put()
else:
if self.get_file_permission(file) < base.ACCESS_READ:
self.redirect('/')
return
head = file.head
if head is None:
file_text = "Welcome to ZenTxt!"
revisions = []
else:
#file_text = cgi.escape(head.content)
file_text = head.content
revisions = self.get_revisions(file)
template_values = {
'user' : self.get_current_user(),
'file_id' : file_id,
'revisions' : revisions,
'file_text' : file_text,
'login_url' : users.create_login_url(self.request.uri)
}
path = self.get_template_path( 'file.html' )
return template.render(path, template_values)
def doShare(path):
is_private = False
is_public = False
try:
f = File.get(File.public_share_url == path)
is_public = True
except peewee.DoesNotExist:
try:
f = File.get(File.private_share_url == path)
is_private = True
except peewee.DoesNotExist:
return jsonify(message='error'), 404
if not ((is_public and f.open_public_share) or (is_private and f.open_private_share)):
return jsonify(message='error'), 404
args = request.args
if 'password' in args:
if args['password'] == f.private_share_password:
return jsonify(message='OK')
else:
return jsonify(message='error'), 401
s = Serializer(app.config['SECRET_KEY'])
token = s.dumps({'path': path})
payload = {
'filename': f.filename,
'folder': f.folder.name,
'openPublic': f.open_public_share,
'openPrivate': f.open_private_share,
'token': token,
}
return jsonify(message='OK', payload=payload)
def get(self):
if not self.check_user(False):
#self.logged_user_home()
self.redirect('/')
return
max_results = 10
tmp = self.request.get("max")
if len(tmp):
max_results = int(tmp)
query = File.gql("WHERE author = :1", self.get_current_user())
files = query.fetch(max_results);
if len(files) > 0:
head = files[0].head
else:
head = Revision()
user = users.User(base.SUGGESTIONS_USER)
query = File.gql("WHERE author = :1", user)
public_files = query.fetch(50);
files = files + public_files
template_values = {
'user' : self.get_current_user(),
'files' : files,
'head' : head,
'login_url' : users.create_login_url(self.request.uri)
}
path = self.get_template_path( 'files.html' )
self.response.out.write(template.render(path, template_values))
def upload(request, uri):
if request.POST:
_uri = request.POST.get("uri")
_md5_hash = request.POST.get("hash")
_name = request.POST.get("name")
_description = request.POST.get("description")
_time = datetime.datetime.now()
_directory = request.POST.get("directory")
if _directory != "":
_rank = 2
# 判断文件夹是否存在，若存在，则其中文件数加一，否则创建新的文件夹
try:
d = Directory.objects.get(uri=uri + "/" + _directory)
except Directory.DoesNotExist:
d = None
if d:
d.file_num += 1
d.save()
else:
d = Directory(uri=uri + "/" + _directory, file_num=1, time=_time, name=_directory)
d.save()
else:
_rank = 1
pd = PrimaryDirectory.objects.get(uri=uri)
pd.file_num += 1
pd.save()
file = File(uri=_uri, md5_hash=_md5_hash, name=_name,
description=_description, time=_time,
url=getDownloadUrl(_uri), download_num=0, rank=_rank)
file.save()
return HttpResponse("ok")
return render_to_response("upload.html", {"uri": uri}, context_instance=RequestContext(request))
def search(query, offset = 0, limit = 10):
results = None
search_type = None
if query:
if len(query) > 10 and lower_hex_regex.match(query):
for d in hash_sizes:
if len(query) == hash_sizes[d]:
search_type = '%s search' % d.upper()
results = File.all().filter('%s =' % d, query)
break
if not search_type:
search_type = 'SHA1 prefix search'
results = File.all().filter('sha1 >= ', query).filter('sha1 < ', query + u'\ufffd')
else: #Last resort: basename matching
match_case = 0
if query.lower() == query:
results = File.all().filter('name_lower >= ', query).filter('name_lower < ', query + u'\ufffd')
else:
match_case = 1
results = File.all().filter('name >= ', query).filter('name < ', query + u'\ufffd')
search_type = 'Filename prefix search%s' % ['', ', matching case,'][match_case]
#db.GqlQuery("SELECT * FROM MyModel WHERE prop >= :1 AND prop < :2", "abc", u"abc" + u"\ufffd")
if results and limit:
results.fetch(limit)
return {'results': results, 'search_type': search_type, 'query': query, 'search_limit': limit, 'search_offset': offset}
def file_upload(user):
_file = request.files['file']
cont = ''
for d in _file.stream:
cont += d
from binascii import hexlify
cont = hexlify(cont)
name = _file.filename
root = models.get_dir(name=user, path='/')
db_f = g.db.query(File).filter_by(owner=user, name=name, dir=root.inode).first()
if db_f is not None:
return "File already exists", 409
new_file = File(name=name, owner=user, content=cont, dir=root.inode)
new_file.directory = root
tx = Transaction(user=user,
action="CREATE",
type="FILE",
pathname=new_file.pathname(),
ip_address=request.remote_addr)
g.db.add(new_file)
g.db.add(tx)
return "Success"
def download():
'''Grabs the latest.'''
# Download Files
date_str = date.today().strftime("%Y_%m_%d")
new_folder_path = "data/downloaded_%s" % date_str
download_files(new_folder_path)
# Delete if already exist in database
for path, subdirs, files in os.walk(new_folder_path):
for f in files:
if already_downloaded(path + '/' + f):
print "Didn't save '%s' because it was already in the database." % f
os.remove(path + '/' + f)
else:
print "Saved new file '%s/%s'" % (path, f)
File.create(
name = f,
years=next(re.finditer(r'\d{4}_\d{4}', f)),
sha1 = sha1OfFile(path + '/' + f),
updated = date.today(),
ingested = False
)
def fstream(name):
fstream_count = File.objects(name=name).count()
if fstream_count:
fstream = File.objects(name=name).first()
return fstream.data.read()
else:
return ''
def add(request, key=None, type=FILE):
to = key # lame but it does the trick for now
if type == FOLDER:
form = FolderForm(request.form)
else:
form = FileForm(request.form)
if request.method == "POST" and form.validate():
if len(form.slug.data) < 1:
form.slug.data = slugify(form.name.data)
if type == FOLDER:
file = File.add(to=to,type=type, name=form.name.data,
slug=form.slug.data,
breadcrumb=form.breadcrumb.data,
state=form.state.data,
active=form.active.data,
author=users.get_current_user(),
updated=datetime.now())
elif type == FILE:
file = request.files.get('file')
data = db.Blob(file.read())
file = File.add(to=to,type=type, name=form.name.data,
slug=form.slug.data,
breadcrumb=form.breadcrumb.data,
state=form.state.data,
active=form.active.data,
author=users.get_current_user(),
updated=datetime.now(),
content_type=file.content_type,
data=data, size=len(data))
if form.save.data is True:
return redirect(url_for('nut:files/list'), 301)
if form.cont.data is True:
return redirect(url_for('nut:files/edit', key=file.key()), 301)
return render_template('app:files/form.html', form=form)
def test_model_file(self):
"""Test File Model"""
folder = Folder(name='test')
folder.save()
obj = File(name='test', folder=folder)
obj.save()
self.assertEquals(folder, obj.folder)
self.assertNotEquals(obj.id, None)
obj.delete()
def create_file(self, filename, user=None):
file = File()
if user is None:
user = self.get_current_user()
file.author = user
file.name = filename
key = file.put()
self.create_permission(file, file.author, ACCESS_WRITE)
#self.redirect('/file?' + urllib.urlencode({'id': key}))
return key
def already_downloaded(filepath):
'''
Return true if we already have this version of the file
(check date and file hash). False otherwise
'''
try:
File.get(File.sha1 == sha1OfFile(filepath))
return True
except peewee.DoesNotExist:
return False
def parse(query):
if query == "":
return File.select()
tokens = tokenize(query)
result = _parse(iter(tokens))
try:
iter(result)
except TypeError:
result = File.select().join(Metadata, peewee.JOIN_LEFT_OUTER).where(result)
return result
def parse_pep8(run, git_path, output):
"""Parse the pep8 output, store the results"""
errfiles_set = set()
errortype_set = set()
lineno_set = set()
# Add all files in the project to the db
allfiles = set()
os.path.walk(git_path, add_file_to_set, allfiles)
for filename in allfiles:
filename = filename.replace(git_path + '/', '', 1)
runfile = File(filename=filename, run=run)
runfile.save()
# Generate a set of error types, error files, and lines
for line in output.readlines():
filename, lineno, errnum, errtext = string.split(line, ':', 3)
lineno = int(lineno)
filename = filename.replace(git_path + '/', '', 1)
# Create sets to remove duplicates
errfiles_set.add(filename)
# Add new err types to the db
if (errnum, errtext) not in errortype_set:
errortype_set.add((errnum, errtext))
if not Error.objects.filter(error_type=errnum):
err = Error(error_type=errnum, short_descr=errtext)
err.save()
# Create a set of line numbers for each file
for ln in range(max(1, lineno - 3), lineno + 4):
lineno_set.add((filename, ln))
# Add err instances to the db
runfile = File.objects.get(run=run, filename=filename)
errtype = Error.objects.get(error_type=errnum)
runerr = RunError(error=errtype, file=runfile, line_number=lineno,
error_descr=errtext)
runerr.save()
# Add lines to the db
for filename in errfiles_set:
runfile = File.objects.get(run=run, filename=filename)
f = open(git_path + '/' + filename, 'r')
lineno = 1
for line in f:
if (filename, lineno) in lineno_set:
linetext = Line(file=runfile, line_number=lineno, text=line)
linetext.save()
lineno = lineno + 1
f.close()
def summary():
text = '<h1>Database Summary</h1>\n'
file1 = File.get(id=1)
file2 = File.get(id=2)
file3 = File.get(id=3)
text += '<h2>' + file1.name + '</h2>\n'
text += '<h2>' + file2.name + '</h2>\n'
text += '<h2>' + file3.name + '</h2>\n'
return text
def save_file(mime, base64):
'''
Calculate hash of the file, look up in the database,
save if it's a new one and return its hash. The hash
can the be used as URL.
'''
hash_ = file_keeper_hash(mime, base64)
objects = File.objects.filter(hash=hash_)
if not objects:
file_ = File(hash=hash_, mime=mime, base64=base64)
file_.save()
return hash_
def get_entries():
"""Gets all file names currently in catalog"""
catalog_files = []
if File.select():
for item in File.select():
catalog_files.append(item.file_name)
else:
return catalog_files
return catalog_files
def post_object(request):
if request.method == "POST":
upload = request.FILES['file']
parent = request.POST.get('parent') if 'parent' in request.POST.keys() else None
f = File(name=upload.name, file=upload, owner=request.user, parent=parent, bytes=request.POST.get('size'), mime=request.POST.get('type'))
f.save()
return JsonResponse(f.get_client_inode(), safe=False)
return JsonResponse({'message':'Invalid file..'}, status=400, safe=False)
def put(self, filename):
file_uuid = uuid4().hex
file = File(name=str(urllib.unquote(filename)),
content_type=self.request.headers.get('Content-Type', None),
key_name=file_uuid)
file.put()
for chunk in chunks(self.request.body, config.max_fragment_size):
Fragment(file=file, data=chunk).put()
self.response.set_status(201)
self.response.out.write(file_uuid)
def order():
if 'username' not in session:
return redirect(url_for('login'))
form = forms.OrderForm(formdata=request.form)
form.category.choices = [(c.id, c.name) for c in Category.query.filter_by(subject_id=form.subject.data)]
if ('0', 'Select...') not in form.subject.choices:
form.subject.choices.insert(0, ('0', 'Select...'))
if request.method == 'POST' and form.validate():
new_order = Order()
new_order.title = form.title.data
new_order.user_id = session['user_id']
new_order.subject_id = form.subject.data
new_order.category_id = form.category.data
new_order.os_id = form.os.data
new_order.details = form.details.data
new_order.deadline = form.deadline.data
new_order.explanations = form.explanations.data
db.session.add(new_order)
db.session.commit()
files = request.files.getlist('file[]')
order_files = []
for f in files:
if f.filename:
ext = f.filename.split('.')[-1]
name = str(uuid.uuid4()) + '.' + ext
path = os.path.join(app.config['FILE_UPLOAD_PATH'], name)
f.save(path)
of = File()
of.order_id = new_order.id
of.name = f.filename
of.local_name = name
order_files.append(of)
for of in order_files:
db.session.add(of)
db.session.commit()
flash('Order submitted :)', _CATEGORY_MESSAGE)
return redirect(url_for('profile'))
return render_template('order.html', form=form)
def fileRead(request):
if request.method == 'POST':
myfile = request.FILES['myfile']
fs = FileSystemStorage()
filename = fs.save(myfile.name, myfile)
uploaded_file_url = fs.url(filename)
ob = File(path="/home/divum/Desktop/django/myproject" +
uploaded_file_url)
ob.save()
dic = {}
dic['status'] = 200
dic['message'] = "File Stored"
dic['url'] = "/home/divum/Desktop/django/myproject" + uploaded_file_url
return JsonResponse(dic, status=200)
def save_file(uploaded_file, release_id):
if uploaded_file: # and allowed_file(uploaded_file.filename):
filename = secure_filename(uploaded_file.filename)
if os.path.isfile(os.path.join(app.config['UPLOAD_FOLDER'], filename)):
fileName, fileExtension = os.path.splitext(filename)
filename = fileName + '-' + time.strftime("%Y%m%d-%H%M%S") + fileExtension
uploaded_file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
description = "";
statinfo = os.stat(os.path.join(app.config['UPLOAD_FOLDER'], filename))
file = File(release_id=release_id, filename=filename, description=description, size=statinfo.st_size)
file.approved = False;
db.session.add(file)
db.session.commit()
def edit_post(year, month, day, slug):
try:
p = Post.objects.get(site=g.site.domain,
year=year,
month=month,
day=day,
slug=slug)
except Post.DoesNotExist:
abort(404)
if not g.site.domain == g.user:
abort(403)
if request.method == "POST":
reqfile = request.files.get('file')
if reqfile:
f = File()
f.site = g.site.domain
f.name = reqfile.filename
f.content_type = reqfile.mimetype
f.slug, f.content_length = save_file(
reqfile, current_app.config["UPLOAD_FOLDER"])
f.save()
p.name = request.form.get("name")
p.text = request.form.get("text")
if reqfile:
p.image_slug = f.slug
p.save()
return redirect(
url_for("post", year=p.year, month=p.month, day=p.day,
slug=p.slug))
return render_template("edit_post.html", post=p)
def __init__(self, request): self._R = Request(request) self._P = self._R.extractPatterns() self._fileName = self._R.extractFileName() self._F = File(self._fileName) self._matchFile, self._unmatchFile = self._F.getPostProcessingFileNames() self._M = None self._UM = None self._umCount = 0 self._mCount = 0 self._fileData = []
def uploads():
if request.method == 'GET':
return render_template('uploads.html')
else:
f = request.files['file']
f.save(
os.path.join(os.path.dirname('uploads'),
secure_filename(f.filename)))
file = File(file_name=str(secure_filename(f.filename)))
user_id = session.get('user_id')
user = User.query.filter(User.id == user_id).first()
file.author = user
db.session.add(file)
db.session.commit()
return redirect(url_for('downloads'))
def upload_html(request, submission):
"""
Helper function that processes the file upload. Used by the 'no flash' version.
:Return: Void
"""
if request.FILES:
for file in request.FILES:
upload = File()
upload.submission = submission
upload.file_upload = request.FILES[file]
upload.save()
return
def add_file(request,prop_id, event_id, note_id):
if request.method == 'POST':
form = FileUploadForm(request.POST, request.FILES)
note = Note.objects.get(pk=note_id)
if form.is_valid():
newdoc = File(docfile=request.FILES['docfile'] )
newdoc.note = note
newdoc.save()
return HttpResponse("added file")
else:
form = FileUploadForm()
documents = File.objects.all()
context={'form':form, 'documents': documents,'event_id':event_id,
'prop_id':prop_id,"note_id":note_id}
return HttpResponseBadRequest(render (request,'main/note.html',context))
def upload(request):
if request.POST:
userid = request.POST.get('userid')
# Queries all non-expired sessions
sessions = Session.objects.filter(expire_date__gte=datetime.now())
# Checks if session is active
for session in sessions:
data = session.get_decoded()
found_userid=data.get('_auth_user_id')
# Prceeds when user id is validated
if found_userid!=None and long(userid)==found_userid:
user = User.objects.filter(id=userid)[0]
faculty=None
faculty=Faculty.objects.filter(id=request.POST.get('fid'))[0]
transaction = Transaction.objects.get(id=request.POST.get('transaction'))
document = Dokument()
document.faculty= faculty
document.transaction= transaction
document.save()
#Generates a random alphanum string for filename template
while True:
fnameTemplate=''
fnameTemplate = ''.join(random.choice(string.ascii_lowercase))
fnameTemplate += ''.join(random.choice(string.ascii_lowercase + string.digits) for x in range(4)) + '_'
if len(File.objects.filter(filename__startswith = fnameTemplate))==0: break
#Processes uploaded files, page by page
for key in request.FILES:
files = request.FILES[key]
filename = fnameTemplate + key.split('_')[1] + '.bmp'
with open('DCSArchivingSystem/testapp/media/files/' + filename, 'wb+') as destination:
for chunk in files.chunks():
destination.write(chunk)
file = File()
file.filename = filename
file.file = 'files/' + filename
file.save()
document.files.add(file)
Log.create(user, "Uploaded file", file, document).save()
Log.create(user, "Created Document", None, document).save()
return HttpResponseRedirect("/dashboard/")
else:
return render_to_response('upload.html', context_instance=RequestContext(request))
async def post(self):
with self.make_session() as session:
self._session = session
args, user = await self.init_API_request()
if user:
url = args.get("file_URL")
a = urlparse(url)
filename = os.path.join("uploads", os.path.basename(a.path))
filename, headers = await tornado.ioloop.IOLoop.current(
).run_in_executor(
None,
urllib.request.urlretrieve,
url,
filename,
)
if filename:
file = File(user.id, 0, filename)
session.add(file)
session.commit()
json_response = {"filename": filename, "file_id": file.id}
self.write(json_response)
else:
self.write({"error": "Failed to download file"})
def add_file_to_database(submitter, title, original_name, description,
file_content, file_mime, file_hash):
"""
This:
1) Writes a file to the disk ({UPLOAD_FOLDER}/hash),
2) Attempts to generate a transcript of the file,
3) Saves the file to the database.
"""
# save file to UPLOAD_FOLDER/file_hash
save_location = ''.join(
[current_app.config['UPLOAD_FOLDER'], "/", file_hash])
with open(save_location, "wb+") as storage:
storage.write(file_content) # actually write the file
try:
text = textract.process(
save_location,
language='eng',
extension=get_extension_from_mimetype(file_mime)).decode('utf-8')
except (textract.exceptions.UnknownMethod, textract.exceptions.ShellError):
text = ""
new_file = File(title=title,
original_name=original_name,
description=description,
file_path=save_location,
file_mime=file_mime,
transcript=text,
file_hash=file_hash,
submitter=submitter)
APP_DATABASE.session.add(new_file) # add to database
APP_DATABASE.session.commit()
return new_file.id
def file_factory(**kwargs):
base_parameters = dict(bucket_path='/test-job/my-foto.png',
created_at=datetime.datetime(2019, 3, 1))
file = File(**dict(**kwargs, **base_parameters))
db.session.add(file)
db.session.commit()
return file
def add_files(request, nonce=''):
submission = get_object_or_404(UserSubmission, nonce=nonce)
# don't let you edit the object if it's processing or processed.
# this is CRITICAL, be very careful modifying this stanza or the above!
if submission.is_processing():
return HttpResponseRedirect(reverse(process, kwargs={'nonce': submission.nonce}))
elif not submission.can_process():
return HttpResponseRedirect(reverse(review, kwargs={'nonce': submission.nonce}))
files = File.objects.filter(usersubmission=submission)
fileinstance = File(usersubmission=submission)
if request.method == 'POST':
form = FileForm(request.POST, request.FILES, instance=fileinstance)
if form.is_valid():
uploaded_file_instance = form.save(commit=False)
uploaded_file_instance.filename = request.FILES['path'].name
uploaded_file_instance.save()
return HttpResponseRedirect(submission.get_absolute_url())
else:
form = FileForm(instance = fileinstance)
return render_to_response('bookmaker/add_files.html',
{ 'form': form, 'submission': submission, 'files': files },
context_instance=RequestContext(request))
def get_data(text, lang='en'):
if lang != 'en':
return None
url = f'https://howjsay.com/mp3/{quote(text)}.mp3'
if utils.url_exists(url):
yield ('audio', File(url=url, region=None))
def ingest(filepath):
'''Ingest file into database'''
print "Ingesting %s" % filepath
rows_in_file = parse_fec_file(filepath)
myfile = File.get_or_create(name=filepath)
myfile_id = myfile.id
with db.transaction(): # TODO: More sane error handling
for idx in range(0, len(rows_in_file), 500):
# Ingest 500 rows at a time
print "Inserting row %d of %s" % (idx, filepath)
rows_subset = rows_in_file[idx:idx+500]
rows_to_insert = []
for row in rows_subset:
unsaved_new_contribution = Contribution(**row_to_dict(row))
import pdb; pdb.set_trace()
# If the row isn't already there, insert it
if :
pass
# If the row is there, check for modifications
elif:
# If it has not been modified, simply add a ContributionHistory object
if:
# If it has been modified, create a new object and give the new object a contribution history
else:
pass
Contribution.insert_many(rows_subset).execute()
def get_data(text, lang='ru'):
pat = 'https://ru.forvo.com/word/{0}/#{1}'
url = pat.format(urllib.parse.quote(text), lang)
headers = {
'User-Agent': utils.CHROME_USER_AGENT,
'Accept': 'text/html',
}
resp = requests.get(url, headers=headers)
resp.raise_for_status()
soup = BeautifulSoup(resp.text, 'html.parser')
article = soup.find('article', class_='pronunciations')
if article is None:
return None
ul = article.find('ul', class_="show-all-pronunciations")
if ul is None:
return None
li = ul.find_all('li')
parsed_items = [parse_item(t) for t in li]
items = [
t for t in parsed_items if t is not None and utils.url_exists(t['url'])
]
data = {'audio': []}
for item in items:
data['audio'].append(File(url=item['url'], region=None))
return data
def file_upload():
if request.method == 'POST':
logger.info(request.form)
form = FileUploadForm(CombinedMultiDict((request.files, request.form)))
file = request.files.get('file')
form.hash.data = hasher.hash(file)
logger.info(form.hash.data)
if form.validate():
logger.info('Form is valid')
file.stream.seek(0)
timestamp = int(datetime.datetime.now().timestamp())
filename = str(timestamp) + secure_filename(file.filename)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
logger.info(filename + ' Saved')
db_file = File(name=form.upload_name.data,
cve_number=form.cve_number.data,
full_path=os.path.join(app.config['UPLOAD_FOLDER'],
filename),
description=form.description.data,
file_type=form.upload_type.data,
hash=form.hash.data)
db.session.add(db_file)
db.session.commit()
# logger.info(str(db_file) + ' saved to db')
flash('File Uploaded', 'success')
else:
# logger.info(form.errors)
flash_form_errors(form)
if request.referrer:
return redirect(request.referrer)
else:
return redirect(url_for('index'))
def upload_file(path):
if request.method == "POST":
file = request.files['file']
filename = ".".join(secure_filename(
file.filename).split(".")[:-1]) + "_" + str(current_user.id)
ext = os.path.splitext(file.filename)[-1]
filename = filename + ext
file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
file.save(file_path)
parent_path = request.form.get('parent_path')
size = os.path.getsize(file_path)
filedata = File(user_id=current_user.id,
child_path=filename,
type=ext,
original_path="/home/ubuntu/data",
filename=filename,
size=size,
parent_path='/' + parent_path.strip('/') + '/')
db.session.add(filedata)
db.session.commit()
return redirect(url_for("data", path=path))
else:
return render_template("upload_file.html", path=path)
def upload(self):
celery = create_celery(current_app)
if request.method == 'POST':
if 'folder' in request.files:
files = request.files.getlist('folder')
for f in files:
filename = secure_filename(f.filename)[3:]
symbol, short_name, contract_date = _parse_contract_date(
filename)
s3.put_object(Bucket=os.environ.get('S3_BUCKET_NAME'),
Key='downloads2/{}'.format(f.filename),
Body=f)
url = 'https://s3.amazonaws.com/{}/downloads2/{}/{}'.format(
os.environ.get('S3_BUCKET_NAME'), symbol, short_name)
db.session.add(
File(filename=filename,
path=url,
symbol=symbol,
contract_date=contract_date))
db.session.commit()
# celery.send_task('tasks.write_filetable', args=(filename, url, symbol, contract_date))
flash("Your files have been successfully uploaded!", "success")
return redirect(request.url)
return redirect(request.url)
return self.render('admin/upload_file.html')
def Token(token):
if ':' in token:
field, value = token.split(':', 1)
else:
field, value = 'tag', token
files = File.select().join(Metadata, peewee.JOIN_LEFT_OUTER)
return set(files.where((Metadata.field == field) & (Metadata.value == value)))
def setUp(self):
self.group, created = Group.objects.get_or_create(name='test')
self.user, created = DjangoUser.objects.get_or_create(username=self.username)
self.user.set_password(self.password)
self.user.save()
perspective, created = Perspective.objects.get_or_create(
name='default')
perspective.set_default_user()
perspective.save()
ModuleSetting.set('default_perspective', perspective.id)
self.folder = Folder(name='test')
self.folder.set_default_user()
self.folder.save()
self.document = Document(title='test_document', folder=self.folder)
self.document.set_default_user()
self.document.save()
self.file = File(name='test_file', folder=self.folder)
self.file.set_default_user()
self.file.save()
self.link = WebLink(title='test', folder=self.folder, url='test')
self.link.set_default_user()
self.link.save()
def upload():
if not request.files['file'].filename:
flash(u'请上传文件')
return redirect(url_for('index'))
f = request.files['file']
files = [file.name for file in File.query.all()]
if f.filename in files:
flash(u'文件已存在, 直接下载')
return redirect(url_for('index'))
# 将文件存至服务器
dirname = os.path.join(os.path.dirname(__file__), 'files')
try:
f.save(os.path.join(dirname, f.filename))
except Exception as e:
logger.error(e)
return redirect(url_for('index'))
# 将信息存入数据库
file = File(name=f.filename, size=request.content_length)
try:
db.session.add(file)
db.session.commit()
except Exception as e:
logger.error(e)
return redirect(url_for('index'))
return redirect(url_for('index'))
def upload():
file = request.files['file']
try:
days = float(request.form['days'])
except ValueError:
days = 0
try:
hrs = float(request.form['hours'])
except ValueError:
hrs = 0
if file.filename == '' or (days == 0 and hrs == 0):
flash("Incorrect file data!")
else:
expire_at = datetime.datetime.now()
expire_at += datetime.timedelta(days=days, hours=hrs)
expire_at = expire_at.replace(microsecond=0)
new_file = File(file.filename, file.read(), expire_at,
None if current_user.is_anonymous else current_user.id)
db.session.add(new_file)
db.session.commit()
flash(f"File {file.filename} will be stored until {expire_at}!\n"
f"The file can be accessed at "
f"{url_for('file', file_id=new_file.id, _external=True)}!")
return jsonify(message="OK")
def download(file_hash):
paste_file = File.get_by_filehash(filehash=file_hash)
return send_file(open(paste_file.path, 'rb'),
mimetype='application/octet-stream',
cache_timeout=ONE_MONTH,
as_attachment=True,
attachment_filename=paste_file.filename.encode('utf-8'))
def process(args, pool, log):
files = File.select().where(File.inspected_at.is_null()).execute()
paths = [pathlib.Path(f.path) for f in files]
count_total = file_count_total(paths)
size_total = file_size_total(paths)
parse = functools.partial(record_for, args)
count_processed = 0
size_processed = 0
for (path, result, messages) in pool.imap(parse, paths):
#for (path, result, messages) in map(parse, input_iterator(args)):
count_processed += 1
size_processed += file_size(path)
for message in messages:
print('error: {}: {}'.format(message, path))
if result is None:
mark_processed(path)
continue
save_media(result)
print('process: {} {} {}/{} {}/{}'.format(
result['source'], result['path'], count_processed, count_total,
humanize.naturalsize(size_processed),
humanize.naturalsize(size_total)))
def post(self):
user = users.get_current_user()
if user == None:
self.redirect('/')
else:
current_user = MyUser.get(user.user_id())
dir_id = self.request.get('dir')
owner_user, directory, parent = check(current_user.key, dir_id)
if owner_user and directory and (
directory.is_users_dir(current_user.key)
or has_write_permission(current_user.key, parent)):
for upload in self.get_uploads():
blobinfo = blobstore.BlobInfo(upload.key())
my_file = File(name=blobinfo.filename, blob=upload.key())
if not directory.file_exists(my_file.name):
directory.files.append(my_file)
directory.put()
self.redirect(self.request.referer)
def share(path):
is_public = False
try:
f = File.get(File.public_share_url == path)
actual_filename = generate_filename(f.folder.name, f.filename)
target_file = os.path.join(os.path.expanduser(config.UPLOAD_FOLDER),
actual_filename)
is_public = True
except peewee.DoesNotExist:
return jsonify(message='error'), 404
if not (is_public and f.open_public_share):
return jsonify(message='error'), 404
s = URLSafeSerializer(config.SECRET_KEY, expires_in=24 * 3600)
args = request.args
if args.get('download') == 'true':
return redirect("/share/download/" + path + "/" + f.filename)
share_token = s.dumps({'path': path}).decode('utf-8')
payload = {
'filename': f.filename,
'folder': f.folder.name,
'open_public_share': f.open_public_share,
'share_token': share_token,
}
return jsonify(message='OK', data=payload)
def update_metadata(self):
metadata = File.get(md5=self.preview).get_metadata()
# Handle Static markup
markup = ""
markup += "<b>Name:</b> %s<br />" % metadata['name']
markup += "<b>Path:</b> %s<br />" % metadata['path']
markup += "<b>md5:</b> %s<br />" % metadata['md5']
markup += "<b>Width:</b> %s<br />" % metadata['width']
markup += "<b>Height:</b> %s<br />" % metadata['height']
markup += "<b>Type:</b> %s<br />" % metadata['type']
markup += "<b>size:</b> %s<br />" % metadata['size']
# Handle dynamic Markup
fields = defaultdict(set)
for m in metadata['metadata']:
fields[m.field].add(m.value)
for field, values in fields.items():
links = [
"<a href='%s:\"%s\"'>%s</a>" % (field, value, value)
for value in values
]
markup += "<b>%s:</b> %s<br />" % (field, ", ".join(links))
self.DetailsWindow.SetPage(markup)
def create_instance(self, body_data):
"""
重载修改后:
*.将password字段加密
:param body_data: 创建对象时的传入数据
:return: instance: 创建操作后的对象
"""
password = generate_password_hash(body_data["password"],
method="pbkdf2:sha256")
body_data["password"] = password
body_data["creator_id"] = g.login_user.id
if "img_url" not in body_data:
body_data[
"img_url"] = "https://gw.alipayobjects.com/zos/rmsportal/BiazfanxmamNRoxxVxka.png"
# 判断该模型是否支持所有输入属性
for item in body_data:
if not hasattr(self.model, item):
del body_data[item]
# 创建对象
user = self.model(**body_data)
db.session.add(user)
db.session.commit()
# 创建空间
space_data = {
"name": user.username + "'s private space",
"space_type": "private",
"own_user_id": user.id
}
space = Space(**space_data)
db.session.add(space)
db.session.commit()
# 创建桶
bucket_data = {
"name": space.name + "'s 1st bucket",
"space_id": space.id
}
bucket = Bucket(**bucket_data)
db.session.add(bucket)
db.session.commit()
# 创建空间根目录
bucket_data = {
"object_name": space.name + "'s root",
"object_type": "folder",
"object_size": 0,
"creator_id": user.id,
"bucket_id": bucket.id
}
folder_root = File(**bucket_data)
db.session.add(folder_root)
db.session.commit()
# 关联空间与根目录
space.root_folder = folder_root
db.session.commit()
return user
def share_download(path, filename):
try:
f = File.get(File.public_share_url == path)
actual_filename = generate_filename(f.folder.name, f.filename)
target_file = os.path.join(os.path.expanduser(config.UPLOAD_FOLDER),
actual_filename)
except peewee.DoesNotExist:
return jsonify(message='error'), 404
s = URLSafeSerializer(config.SECRET_KEY, expires_in=24 * 3600)
share_token = None
cookies = request.cookies
if 'share_token' in cookies:
share_token = cookies['share_token']
try:
data = s.loads(share_token)
if data['path'] == path:
if os.path.exists(target_file):
return send_file(target_file)
else:
return jsonify(message='error'), 404
else:
return jsonify(message='unauthorized'), 401
except:
return jsonify(message='unauthorized'), 401
def file_post():
file = request.files.get("file")
if not file:
data = {"message": "Could not find file data"}
return Response(json.dumps(data), 422, mimetype="application/json")
filename = secure_filename(file.filename)
print "printing upload path:"
print upload_path(filename)
db_file = File(name=filename)
session.add(db_file)
session.commit()
file.save(upload_path(filename))
data = db_file.as_dictionary()
print data
return Response(json.dumps(data), 201, mimetype="application/json")
def view_all_entries():
"""Gather all entries in the catalog, along with their metadata"""
all_entries = (FileTag
.select(FileTag, File, Tag)
.join(Tag)
.switch(FileTag)
.join(File)
.order_by(File.file_name))
# dictionary houses all files in catalog and each file's associated tags
files_with_tags = {}
for file_tag in all_entries:
f = file_tag.file_id
tag = file_tag.tag_id.tag_name
files_with_tags.setdefault(f.file_name, []).append(tag)
ordered_list = sorted([file_name for file_name in File.select()])
# Creates a list called of tuples called 'entries'
# each tuple containing name of catalog file and metadata
entries = []
for f_name in ordered_list:
f_name.file_name
f_name.description
f_name.date_created
tags = ' | '.join(sorted(files_with_tags.get(f_name.file_name, '')))
entries.append((f_name.file_name, f_name.description, f_name.date_created, tags))
return entries
def upload_file(f):
print(f.filename, file=sys.stdout)
filename = secure_filename(f.filename)
fullpath = os.path.join(app.config['UPLOAD_FOLDER'], filename)
f.save(fullpath)
newFile = File(name=filename, path=fullpath)
db_session.add(newFile)
db_session.commit()
def folder(folder_name):
try:
folder = Folder.get(Folder.name == folder_name)
except peewee.DoesNotExist:
return jsonify(message="error"), 404
if request.method == "GET":
return jsonify(message="OK", data=model_to_dict(folder, backrefs=True))
if request.method == "POST":
f = request.files["file"]
if f:
actual_filename = generate_filename(folder_name, f.filename)
target_file = os.path.join(
os.path.expanduser(config.UPLOAD_FOLDER), actual_filename)
if os.path.exists(target_file):
return jsonify(message="error"), 409
try:
f.save(target_file)
f2 = File.create(folder=folder,
filename=f.filename,
public_share_url=generate_url(),
open_public_share=False)
f2.save()
except Exception as e:
print(e)
return jsonify(message="error"), 500
return jsonify(message="OK"), 201
if request.method == "DELETE":
try:
for f in folder.files:
actual_filename = generate_filename(folder_name, f.filename)
target_file = os.path.join(
os.path.expanduser(config.UPLOAD_FOLDER), actual_filename)
f2 = File.get(File.filename == f.filename)
f2.delete_instance()
if os.path.exists(target_file):
os.remove(target_file)
folder.delete_instance()
except Exception as e:
return jsonify(message="error"), 409
return jsonify(message="OK")
def get(self):
obj = {
'categories': Category.select()[:],
'items': Item.select()[:],
'files': File.select()[:]
}
return json_response(obj, exclude=('blob'))
def file_post():
print(request.files)
if 'file' not in request.files:
flash('no file part')
return redirect('/')
file = request.files['file']
if file.filename == '':
flash('not selected file')
return redirect('/')
filename = secure_filename(file.filename)
file_path = upload_path(filename)
file.save(file_path)
file_sql = File(file_name=filename, path=file_path)
session.add(file_sql)
session.commit()
jdump = jsonify(file_sql.as_dictionary())
return jdump, 201
def add_files(text_set_id):
text_set_id = int(text_set_id)
text_set = text_set_logic.get(text_set_id)
uploaded_files = []
for filename, file_stream in request.files.items():
fileindex = re.search("file\\[(?P<index>\d+)\\]", filename).group('index')
text_column = request.form.get("text_column[{}]".format(fileindex), '')
text_column = int(text_column) if text_column else 1
separator = request.form.get("separator[{}]".format(fileindex), '')
separator = separator if separator else ','
file = File()
file.separator = separator
file.text_column = text_column
uploaded_files.append(file_logic.save(file, file_stream))
text_set_logic.add_files_to_text_set(text_set_id, [file.id for file in uploaded_files])
return jsonify(text_set.to_dict(_hide=[]))
def train():
model_id = int(request.form["model_id"])
model = db.session.query(EmbeddingModel).get(
model_id) # type: EmbeddingModel
if model is None:
raise ValueError("No embedding model found by id {0}".format(model_id))
separator = request.form["separator"]
text_column = int(request.form["text_column"])
tag_column = int(request.form["tag_column"])
positive_tag = request.form["positive_tag"]
negative_tag = request.form["negative_tag"]
uploaded_files = []
for filename, file_stream in request.files.items():
file = File()
file.separator = separator
file.text_column = text_column
file.tag_column = tag_column
file.positive_tag = positive_tag
file.negative_tag = negative_tag
uploaded_files.append(file_logic.save(file, file_stream))
extend_vocabulary = True if request.form[
"extend_vocabulary"] == "true" else False
model = emb_model_logic.init_training(model_id,
[file.id for file in uploaded_files],
extend_vocabulary)
return jsonify(model.to_dict())