def save_files(db, user, files, folder):
dir_path = os.path.join('files', user, folder)
if not os.path.exists(dir_path):
try:
form = schemas.Folder_create
form.folder_name = folder
create_folder(db, user, form)
except Exception as e:
print(e)
try:
for uploaded_file in files:
file_name = uploaded_file.filename
file_path = os.path.join(dir_path, file_name)
with open(file_path, "wb+") as file_object:
file_object.write(uploaded_file.file.read())
new_file = models.File(
file_name=file_name,
folder=folder,
username=user,
)
db.add(new_file)
db.commit()
except Exception as e:
print(e)
return False
return True
def test_inherit():
# TODO this test depends on particular repo_models modules and fields
collection = models.Collection(path_abs='/var/www/media/ddr/ddr-test-123')
entity = models.Entity(
path_abs='/var/www/media/ddr/ddr-test-123/files/ddr-test-123-456')
file_ = models.File(
path_abs=
'/var/www/media/ddr/ddr-test-123/files/ddr-test-123-456/files/ddr-test-123-456-master-abc123'
)
collection.public = True
entity.public = False
assert collection.public == True
assert entity.public == False
inheritance.inherit(collection, entity)
assert collection.public == True
assert entity.public == True
entity.public = True
file_.public = False
assert entity.public == True
assert file_.public == False
inheritance.inherit(entity, file_)
assert entity.public == True
assert file_.public == True
collection.public = True
file_.public = False
assert collection.public == True
assert file_.public == False
inheritance.inherit(collection, file_)
assert collection.public == True
assert file_.public == True
def upload():
# if request.method == 'POST':
# file = request.files('file')
# newfile = models.Document(1,'cent0594','no title',filename = file.filenane, doc=file.read())
# models.db.session.add(newfile)
# models.db.session.commit()
if request.method == 'POST':
# check if the post request has the file part
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
file = request.files['file']
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
flash('No selected file')
return redirect(request.url)
if file: # and allowed_file(file.filename):
filename = secure_filename(file.filename)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
newfile = models.File(filename=file.filename,
page=models.Entry.query.first())
# newfile = models.FileContents(name=file.filename, data=file.read(),mimetype=file.mimetype,page = models.Entry.query.first())
models.db.session.add(newfile)
models.db.session.commit()
# return redirect(url_for('uploaded_file',
# filename=filename))
return render_template("upload.html")
def song_post():
""" post a song """
headers = {
"Location": url_for("song_post"),
"Content-Type": "application/json"
}
######### get the data from the form
data = request.json
post_song = models.Song(
song_name=data["song_name"],
id=data["song_id"]) ## ask sam if we really need to post seperately.
post_file = models.File(song_id=data["song_id"],
file_name=data["file_name"],
id=data["file_id"])
if not session.query(models.Song).get(
post_song.id
): #consider adding a check here for duplicate fileID too
session.add_all([post_song, post_file])
session.commit()
else:
print "************* ELSE ***************"
session.rollback()
session.flush()
return Response(json.dumps(
{"status": "failed - that song already exists"}),
500,
mimetype="application/json")
return Response(stripUnicode(data),
200,
headers=headers,
mimetype="application/json")
def addMedia(request):
'''
takes {
id: oshash
filename: string,
item: string
info: {}
}
returns {
item: id,
}
'''
response = json_response({})
data = json.loads(request.POST['data'])
oshash = data.pop('id')
if not request.user.get_profile().capability('canAddItems'):
response = json_response(status=403, text='permissino denied')
elif models.File.objects.filter(oshash=oshash).count() > 0:
f = models.File.objects.get(oshash=oshash)
if f.available:
response['status']['text'] = 'file exists'
response['data']['item'] = f.item.itemId
response['data']['itemUrl'] = request.build_absolute_uri('/%s' %
f.item.itemId)
else:
if 'item' in data:
i = Item.objects.get(itemId=data['item'])
else:
title = ox.parse_movie_path(os.path.splitext(
data['filename'])[0])['title']
i = Item()
i.data = {
'title': title,
'director': data.get('director', []),
}
i.user = request.user
i.save()
i.make_poster(True)
f = models.File(oshash=oshash, item=i)
f.path = data.get('filename', 'Untitled')
extension = f.path.split('.')
if len(extension) > 1:
extension = extension[-1]
else:
#wafaa
#extension = 'webm'
extension = 'png'
f.selected = True
if 'info' in data and data['info']:
f.info = data['info']
f.info['extension'] = extension
f.parse_info()
f.save()
response['data']['item'] = i.itemId
response['data']['itemUrl'] = request.build_absolute_uri('/%s' %
i.itemId)
return render_to_json_response(response)
def commit():
req = json.loads(request.data)
if req.get('safe_token') != gcfg.safe.token:
return flask.jsonify(dict(content=None))
job_id = int(req.get('id'))
job = models.Job[job_id]
job.updated = datetime.datetime.today()
#job.output = req.get('output')
job.version = req.get('version')
job.gobuildrc = req.get('gobuildrc')
success = req.get('success', False)
job.status = 'finished' if success else 'error'
print 'job-status:', job.id, job.status
build = job.build
build.lastest_job = job.id
build.status = job.status
if req.get('success', False):
build.downloadable = True
build.time_used = req.get('time_used')
build.version = req.get('version')
build.repo.updated = build.updated = job.updated
build.repo.downloadable = True
osarch_map = {}
for osarch, ds in req.get('files').items():
goos, arch = osarch.strip().split('_')
archs = osarch_map.get(goos, [])
archs.append(arch)
osarch_map[goos] = archs
file = models.File.get(build=build, os=goos, arch=arch) or \
models.File(build=build, os=goos, arch=arch, reponame=build.repo.name)
file.loglink = ds.get('loglink', '')
file.outlink = ds.get('outlink')
file.size = ds.get('size')
file.md5 = ds.get('md5')
file.sha = ds.get('sha')
#print osarch, ds
# store in format: [{"windows": ["386", "amd64"]},..]
store_list = []
for goos in 'windows', 'linux', 'darwin':
archs = osarch_map.get(goos)
if archs:
archs.sort()
store_list.append([goos, archs])
build.osarchs = json.dumps(store_list)
return flask.jsonify(dict(status=0, message='success'))
def get_or_create_file(volume, f, user, item=None):
try:
file = models.File.objects.get(oshash=f['oshash'])
except models.File.DoesNotExist:
file = models.File()
file.oshash = f['oshash']
file.path = f['path']
if item:
file.item = item
else:
file.item = None #gets pupulated later via update_info
file.save()
return file
def analyse_song():
file = request.files.get("file")
if file.id == "<id>":
pass
if file.id != "<id>":
data = {"message": "Wrong file id"}
return Response(json.dumps(data), 422, mimetype="application/json")
filename = secure_filename(file.filename)
db_file = models.File(filename=filename)
# pass in the path of the uploaded file
analyse(upload_path(filename))
return Response(json.dumps(data), 201, mimetype="application/json")
def file_post():
file = request.files.get("file")
if not file:
data = {"message": "Could not find file data"}
return Response(json.dumps(data), 422, mimetype="application/json")
filename = secure_filename(file.filename)
db_file = models.File(filename=filename)
session.add(db_file)
session.commit()
file.save(upload_path(filename))
data = db_file.as_dictionary()
return Response(json.dumps(data), 201, mimetype="application/json")
def file_editor(file, language, username):
''' This view shows the sentences in a file to edit
:Parameters:
- 'file': the file currently being edited
- 'language': the current target language
- 'username': the current user
'''
if file[-1] == '/':
file = file[:-1]
f = models.File(oid=models.find_file("en", language, file)['_id'])
u = models.User(username=username)
if f.grab_lock(u._id) == False:
return redirect('/file/{0}/{1}/{2}/423'.format(username,language,file))
sentences = models.get_sentences_in_file(urllib.url2pathname(file), 'en', language)
return render_template('file_editor.html', sentence_list=sentences, language=language, username=username)
def receive(self, incoming_message):
sender_email = re.search('.*<(.*)>', incoming_message.sender)
if not sender_email:
self.error(httplib.BAD_REQUEST)
logging.error('Could not extract email from: %s',
incoming_message.sender)
return
sender_email = sender_email.group(1)
file_entities = []
for attachment in getattr(incoming_message, 'attachments', []):
file_entities.append(models.File(
id=_generate_id(),
user_email=sender_email,
filename=attachment.filename,
payload=attachment.payload.decode()))
if not file_entities:
logging.info('Email from %s contained no attachments.',
sender_email)
outgoing_message = mail.EmailMessage(
sender=MAIL_SENDER,
to=sender_email,
subject="You didn't send us any files.",
body=('You asked us to save some files for you, '
'but you did not attach any in your email.'))
outgoing_message.send()
return
ndb.put_multi(file_entities)
logging.info('Saved %d file(s) for %s.',
len(file_entities), sender_email)
file_names = [entity.filename for entity in file_entities]
file_urls = [' * {}: {}/view/{}'.format(
entity.filename,
app_identity.get_default_version_hostname(),
entity.key.id())
for entity in file_entities]
outgoing_message = mail.EmailMessage(
sender=MAIL_SENDER,
to=sender_email,
subject='We received your files: {}'.format(', '.join(file_names)),
body='You can access them at the following locations:\n\n{}'.format(
'\n'.join(file_urls)))
outgoing_message.send()
def write_mongo(po_fn, userID, status, language, po_root, db):
'''write a po_file to mongodb
:Parameters:
- 'po_fn': the file name of the current pofile
- 'userID': the ID of the user that translated the po file
- 'status': the status of the translations
- 'language': The target_language of the translations (source assumed to be english)
- 'po_root': the root of the po_files
'''
po = polib.pofile(po_fn)
rel_fn = os.path.relpath(po_fn, po_root)
rel_fn = os.path.splitext(rel_fn)[0]
f = models.File(
{
u'file_path': rel_fn,
u'priority': 0,
u'source_language': u'en',
u'target_language': language
},
curr_db=db)
i = 0
reg = re.compile('^:[a-zA-Z0-9]+:`(?!.*<.*>.*)[^`]*`$')
for entry in po.translated_entries():
sentence_status = status
match = re.match(reg, entry.msgstr.encode('utf-8'))
if match is not None and match.group() == entry.msgstr.encode('utf-8'):
sentence_status = "approved"
t = models.Sentence(
{
u'source_language': u'en',
u'source_sentence': entry.msgid.encode('utf-8'),
u'sentenceID': entry.tcomment.encode('utf-8'),
u'sentence_num': i,
u'fileID': f._id,
u'target_sentence': entry.msgstr.encode('utf-8'),
u'target_language': language,
u'userID': userID,
u'status': sentence_status,
u'update_number': 0
},
curr_db=db)
t.save()
i += 1
f.get_num_sentences()
def file_add(request):
if request.method == 'POST':
form = forms.FileAddForm(request.POST, request.FILES)
if form.is_valid():
file = request.FILES['file']
filename = file.name
filepath = os.path.join(settings.MEDIA_ROOT, filename)
#myfile = File(file=SimpleUploadedFile(filename, default_storage.open(filename).read()))
#myfile = models.File(file=SimpleUploadedFile(filename, open(filepath).read())) # unicode error
myfile = models.File(file=file) # unicode error
myfile.save()
return redirect('filer.views.file_list')
else:
form = forms.FileAddForm()
return render_to_response('filer/form.html',
context_instance=RequestContext(
request, {
'form': form,
}))
def add_file(original, translated):
success = False
error = ''
data = []
try:
new_file = models.File(original, translated)
db.session.add(new_file)
db.session.commit()
db.session.refresh(new_file)
data = {
'id': new_file.id,
'original_file': original,
'translated': translated
}
success = True
except Exception as e:
error = str(e)
finally:
return {'success': success, 'data': data, 'error': error}
def file_post():
# Attempt to obtain the file uploaded file from Flask's request.files dict
file = request.files.get("file")
# If the file is not found, return an error
if not file:
data = {"message": "Could not find file data"}
return Response(json.dumps(data), 422, mimetype="application/json")
# Werkzeug secure_filename function provides safe version of file name
# For instance ../../../etc/passwd is replaced by etc_passwd
filename = secure_filename(file.filename)
# Create file object with safe filename
db_file = models.File(name=filename)
# Add the file object to the session and commit
session.add(db_file)
session.commit()
# Save the file to the upload path using the safe file name
file.save(upload_path(filename))
# Create a dict object of file
data = db_file.as_dictionary()
# Return a response with 201 CREATED
return Response(json.dumps(data), 201, mimetype="application/json")
def upload(request):
if request.method == 'POST':
uploadform = forms.FileUploadForm(request.POST, request.FILES)
if uploadform.is_valid():
#file = FileField.objects.create()
x = get_random_string(12)
newfile = models.File(file_id=x,
file=request.FILES['file'],
owner=request.user,
name=request.POST['name'],
description=request.POST['description'],
date_time=datetime.datetime.now(),
ip_address=get_client_ip(request))
newfile.save()
#uploadform.save()
#return file(request, x)
return redirect('/file/%s' % x)
else:
uploadform = forms.FileUploadForm()
args = {}
args['form'] = uploadform
return render(request, 'upload/upload.html', context=args)
def song_delete():
""" delete a song """
headers = {"Location": url_for("song_delete")}
# get the data from the form
data = request.json
post_song = models.Song(
song_name=data["song_name"],
id=data["song_id"]) ## ask sam if we really need to post seperately.
post_file = models.File(song_id=data["song_id"],
file_name=data["file_name"],
id=data["file_id"])
if session.query(models.Song).get(
post_song.id
): #consider adding a check here for duplicate fileID too
del_song = session.query(models.Song).get(post_song.id)
session.delete(del_song)
del_file = session.query(models.File).get(post_file.id)
session.delete(del_file)
session.commit()
else:
print "************* ELSE ***************"
session.rollback()
session.flush()
return Response(json.dumps(
{"status": "failed - that song doesnt exists"}),
500,
mimetype="application/json")
return Response(json.dumps({"status": "deleted"}),
200,
headers=headers,
mimetype="application/json")
def add_file(self, upfile):
# get content
file = tempfile.NamedTemporaryFile(delete=False)
upfile.save(file)
file.flush()
file.close()
# get md5
md5 = hashlib.md5(file.name).hexdigest()
levels = self.get_sublevels(md5)
# get current sublevel
storage_path = ''
# create sublevels
dir = self.storage_path
for level in levels:
dir = os.path.join(dir, level)
storage_path = os.path.join(storage_path, level)
if not os.path.exists(dir):
os.makedirs(dir)
dir = os.path.join(dir, md5)
storage_path = os.path.join(storage_path, md5)
if not os.path.exists(dir):
os.makedirs(dir)
storage_path = os.path.join(storage_path, upfile.filename)
# copy file
shutil.copyfile(file.name, os.path.join(dir, upfile.filename))
#TODO: Delete file.name from temp storage
# add file to db
file = models.File(source_name=upfile.filename, storage_path=storage_path.replace('\\','/'))
file.save()
print "Add file", upfile.filename, "as", storage_path
return file
def file_post():
#try to access uploaded file from Flask's request.files dictionary
file = request.files.get("file")
if not file:
data = {"message": "Could not find file data"}
return Response(json.dumps(data), 422, mimetype="application/json")
#secure_filename() --> function(from Werkzeug) which creates a safe version of the filename supplied by client
filename = secure_filename(file.filename)
#use the secure filename to create a File object and add it to the database and commit
db_file = models.File(filename=filename)
session.add(db_file)
session.commit()
#save file to an upload folder using our upload_path() function
file.save(upload_path(filename))
#return file information
data = db_file.as_dictionary()
"""View how data above looks
print ("file data from file_post() is {}".format(data))
print ("")
print("Response data is {}".format(json.dumps(data)))
"""
return Response(json.dumps(data), 201, mimetype="application/json")
def _create_or_edit(entry, template, topics, tags):
if request.method == 'POST':
entry.title = request.form.get('title') or ''
entry.content = request.form.get('content') or ''
entry.published = request.form.get('published') or False
if not (entry.title and entry.content):
flash('Title and Content are required.', 'danger')
else:
# Wrap the call to save in a transaction so we can roll it back
# cleanly in the event of an integrity error.
ret = entry.save()
if not ret:
flash('Warning: this title is already in use.', 'warning')
else:
flash('Entry saved successfully.', 'success')
for t in entry.topics:
entry.topics.remove(t)
models.db.session.commit()
t = models.Topic.query.get_or_404(int(
request.form.get('topic_id')))
entry.topics.append(t)
models.db.session.add(entry)
for t in entry.tags:
entry.tags.remove(t)
models.db.session.commit()
for tagid in request.form.getlist('tag_id'):
t = models.PageTag(int(tagid), entry.id)
models.db.session.add(t)
models.db.session.commit()
for f in entry.documents:
delete_this_entry = request.form.get(
('delete_{}').format(f.id)) or False
if delete_this_entry:
models.db.session.delete(f)
os.remove(
os.path.join(app.config['UPLOAD_FOLDER'], f.filename))
models.db.session.commit()
# check if the post request has the file part
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
file = request.files['file']
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
flash('No selected file')
return redirect(request.url)
if file: # and allowed_file(file.filename):
filename = ("{}_{}").format(entry.id,
secure_filename(file.filename))
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
caption = request.form.get('caption')
f = models.File.query.filter_by(filename=file.filename)
if f.count() > 0:
flash('file already exists and is being overwritten')
newfile = f.first()
newfile.caption = caption
else:
newfile = models.File(filename=filename,
page=entry,
caption=caption)
# newfile = models.FileContents(name=file.filename, data=file.read(),mimetype=file.mimetype,page = models.Entry.query.first())
models.db.session.add(newfile)
models.db.session.commit()
# return redirect(url_for('uploaded_file',
# filename=filename))
if entry.published:
return redirect(url_for('detail', slug=entry.slug))
else:
return redirect(url_for('edit', slug=entry.slug))
# try:
# # with database.atomic():
# entry.save()
# except IntegrityError:
# flash('Error: this title is already in use.', 'danger')
# else:
# flash('Entry saved successfully.', 'success')
# if entry.published:
# return redirect(url_for('detail', slug=entry.slug))
# else:
# return redirect(url_for('edit', slug=entry.slug))
return render_template(template, entry=entry, topics=topics, tags=tags)
def file(self, id=None):
'''This method wraps around the file creator to provide the correct db'''
f = models.File(oid=id, curr_db=self.db)
return f
class UploadHandler(webapp2.RequestHandler):
def getSize(self, fileobject):
fileobject.seek(0, 2) # move the cursor to the end of the file
size = fileobject.tell()
return size
def get(self):
template = main.jinja_env.get_template("templates/upload.html")
values = {
"departments": models.DEPARTMENT_NAMES,
"depts": models.DEPARTMENTS,
"types": models.FILE_TYPES
}
termcodes = []
this_year = date.today().year + 1
this_month = date.today().month
for year in range(this_year, this_year - 3, -1):
yearcode = year * 100
for quarter, value in [('Spring', 30), ("Winter", 20),
("Fall", 10)]:
termcodes.append({
"term":
quarter + " " + str(year - 1) + "-" + str(year),
"code":
yearcode + value
})
values["termcodes"] = termcodes
if 1 <= this_month <= 2:
# Winter Quarter
values["termcodes"] = termcodes[4:]
elif 3 <= this_month <= 6:
# Spring Quarter
values["termcodes"] = termcodes[3:]
elif 7 <= this_month <= 8:
# Summer
pass
elif 9 <= this_month <= 11:
# Fall Quarter
values["termcodes"] = termcodes[2:]
elif this_month == 12:
# Winter Quarter
values["termcodes"] = termcodes[1:]
hash = self.request.params.get('msg')
if hash:
if hash == 'success':
values[
'msg'] = 'Congrats! The file was uploaded. The chapter applauds your committment to academic success.'
elif hash == 'retry':
values[
'msg'] = 'The file needs to be less than 10MB or the request was bad. Please try submitting again, or shrinking the file down.'
elif hash == 'error':
values[
'msg'] = 'Error uploading file! Please contact Tech if this is a problem'
self.response.out.write(template.render(values))
def post(self):
client = gdata.sites.client.SitesClient(source='rosefiji-fijifiles-v1',
site='fiji-files',
domain='rosefiji.com')
import os.path
folder = os.path.dirname(os.path.realpath(__file__))
file_path = os.path.join(folder, 'credentials.json')
credentials = ServiceAccountCredentials.from_json_keyfile_name(
file_path, scopes)
credentials._kwargs['sub'] = '*****@*****.**'
auth2token = gdata.gauth.OAuth2TokenFromCredentials(credentials)
auth2token.authorize(client)
dept = self.request.params.get('department')
file_id = models.get_upload_number()
uri = '%s?kind=%s' % (client.MakeContentFeedUri(),
'filecabinet&path=/' + dept)
feed = client.GetContentFeed(uri=uri)
f = self.request.params.get('file')
ext = f.filename.split(".")[-1].upper()
media = gdata.data.MediaSource(
file_handle=f.file.read(),
content_type=gdata.docs.service.SUPPORTED_FILETYPES.get(ext)
or 'application/octet-stream',
content_length=self.getSize(self.request.params.get('file').file))
try:
urlfetch.set_default_fetch_deadline(30)
attachment = client.UploadAttachment(
media,
feed.entry[0],
title=file_id,
description='UPLOADED TO NEW FIJI FILES SITE')
except Exception, e:
logging.exception(e)
self.redirect('/upload?msg=retry')
return
try:
course_dept = dept
course_number = int(self.request.params.get("courseNumber"))
course_key = ndb.Key("Course", course_dept + str(course_number))
course = course_key.get()
if not course:
course = models.Course(key=course_key,
department=course_dept,
course_number=course_number)
course.put()
file_to_add = models.File(
parent=course_key,
professor=self.request.params.get("professor"),
file_type=self.request.params.get("type"),
termcode=int(self.request.params.get("termcode")),
url="https://sites.google.com/a/rosefiji.com/fiji-files/" +
dept + "/" + file_id,
delete_url=str(attachment.get_edit_link().href),
comments=self.request.params.get("comments"))
file_to_add.put()
self.redirect('/upload?msg=success')
except Exception, e:
logging.exception(e)
self.redirect('/upload?msg=error')
