Beispiel #1
0
def RegisterFile(Service=None, FileName=None, ProvisionedSpace="10G"):

    if Service  is None:
	raise StorageError('RegisterFile(): Service can not be None')

    if FileName is None:
	raise StorageError('RegisterFile(): FileName can not be None')

    vfilespace = StringSizeToBytes(ProvisionedSpace)
    
    if Service.freespace - vfilespace > 0:
	NewFile = File()
	NewFile.vfilename 	= FileName
        NewFile.ufid		= GetUniqueFileID(FileName)
	NewFile.pfilesize	= 0
        NewFile.vfilesize	= vfilespace
	NewFile.service		= Service
	NewFile.pfilename	= GetPhysicalFileName(Service.localpath, FileName)
	NewFile.status		= 'O'
	NewFile.save()
	
	SFreeSpace = CalculateFreeSpace(Service)
	Service.freespace = SFreeSpace
	Service.save()

	return NewFile
    else:
	raise StorageError('RegisterFile(): No have left space')
Beispiel #2
0
def expose_path(group):
    form_keys = ["path", "size", "hash", "hash_function", "modified", "signature"]
    for key in form_keys:
        if key not in request.form:
            raise ValueError("Missing form value %s" % key)
    path = request.form["path"]
    signature = request.form["signature"]

    # Don't include signature
    d = {k: request.form[k] for k in form_keys[:-1]}

    if not security.check_json_sig(d, app.config["BASEJUMP_KEY"], signature):
        raise ValueError("Invalid signature provided.")

    key = security.sign_path(path, app.config["SECRET_KEY"])

    meta = request.form
    with db_session() as s:
        f = s.query(File).filter(File.key == key).all()
        if f:
            raise ValueError("This path is already exposed.")

        last_modified = datetime.fromtimestamp(int(meta["modified"]))
        f = File(path=path, group=group, key=key, size=meta["size"], checksum=meta["hash"], checksumType=meta["hash_function"], modified=last_modified)
        s.add(f)
        s.commit()
        url_path = f.queue_url()
    url = request.url_root + url_path
    return jsonify({"queue_url": url})
Beispiel #3
0
def file_path(slug):
    f = File.all().filter('abs_path = ', slug).get()
    if f is None:
        f = File.all().filter('slug = ', slug).get()
    if f is None:
        return u''
    return f.get_absolute_url()
Beispiel #4
0
    def GetOutput(self):
#        if not self.check_user():
#            return

        file_id = self.request.get("id")
        file = self.get_file(file_id)
        if file is None:
            file = File()
            file.put()
        else:
            if self.get_file_permission(file) < base.ACCESS_READ:
                self.redirect('/')
                return

        head = file.head
        if head is None:
            file_text = "Welcome to ZenTxt!"
            revisions = []
        else:
            #file_text = cgi.escape(head.content)
            file_text = head.content
            revisions = self.get_revisions(file)

        template_values = {
            'user'      : self.get_current_user(),
            'file_id'   : file_id,
            'revisions' : revisions,
            'file_text' : file_text,
            'login_url' : users.create_login_url(self.request.uri)
        }

        path = self.get_template_path( 'file.html' )
        return template.render(path, template_values)
Beispiel #5
0
def doShare(path):
    is_private = False
    is_public = False

    try:
        f = File.get(File.public_share_url == path)
        is_public = True
    except peewee.DoesNotExist:
        try:
            f = File.get(File.private_share_url == path)
            is_private = True
        except peewee.DoesNotExist:
            return jsonify(message='error'), 404

    if not ((is_public and f.open_public_share) or (is_private and f.open_private_share)):
        return jsonify(message='error'), 404

    args = request.args
    if 'password' in args:
        if args['password'] == f.private_share_password:
            return jsonify(message='OK')
        else:
            return jsonify(message='error'), 401

    s = Serializer(app.config['SECRET_KEY'])
    token = s.dumps({'path': path})

    payload = {
        'filename': f.filename,
        'folder': f.folder.name,
        'openPublic': f.open_public_share,
        'openPrivate': f.open_private_share,
        'token': token,
    }
    return jsonify(message='OK', payload=payload)
Beispiel #6
0
    def get(self):
        if not self.check_user(False):
            #self.logged_user_home()
            self.redirect('/')
            return

        max_results = 10
        tmp = self.request.get("max")
        if len(tmp):
            max_results = int(tmp)

        query = File.gql("WHERE author = :1", self.get_current_user())
        files = query.fetch(max_results);

        if len(files) > 0:
            head = files[0].head
        else:
            head = Revision()

        user = users.User(base.SUGGESTIONS_USER)
        query = File.gql("WHERE author = :1", user)
        public_files = query.fetch(50);
        files = files + public_files

        template_values = {
            'user'      : self.get_current_user(),
            'files'   	: files,
            'head'      : head,
            'login_url' : users.create_login_url(self.request.uri)
        }

        path = self.get_template_path( 'files.html' )
        self.response.out.write(template.render(path, template_values))
Beispiel #7
0
def upload(request, uri):
    if request.POST:
        _uri = request.POST.get("uri")
        _md5_hash = request.POST.get("hash")
        _name = request.POST.get("name")
        _description = request.POST.get("description")
        _time = datetime.datetime.now()
        _directory = request.POST.get("directory")
        if _directory != "":
            _rank = 2
            # 判断文件夹是否存在,若存在,则其中文件数加一,否则创建新的文件夹
            try:
                d = Directory.objects.get(uri=uri + "/" + _directory)
            except Directory.DoesNotExist:
                d = None
            if d:
                d.file_num += 1
                d.save()
            else:
                d = Directory(uri=uri + "/" + _directory, file_num=1, time=_time, name=_directory)
                d.save()
        else:
            _rank = 1
        pd = PrimaryDirectory.objects.get(uri=uri)
        pd.file_num += 1
        pd.save()
        file = File(uri=_uri, md5_hash=_md5_hash, name=_name,
                    description=_description, time=_time,
                    url=getDownloadUrl(_uri), download_num=0, rank=_rank)
        file.save()
        return HttpResponse("ok")
    return render_to_response("upload.html", {"uri": uri}, context_instance=RequestContext(request))
Beispiel #8
0
def search(query, offset = 0, limit = 10):
  results = None
  search_type = None

  if query:
    if len(query) > 10 and lower_hex_regex.match(query):
      for d in hash_sizes:
        if len(query) == hash_sizes[d]:
          search_type = '%s search' % d.upper()
          results = File.all().filter('%s =' % d, query)
          break
      if not search_type:
        search_type = 'SHA1 prefix search'
        results = File.all().filter('sha1 >= ', query).filter('sha1 < ', query + u'\ufffd')
    else: #Last resort: basename matching
      match_case = 0
      if query.lower() == query:
        results = File.all().filter('name_lower >= ', query).filter('name_lower < ', query + u'\ufffd')
      else:
        match_case = 1
        results = File.all().filter('name >= ', query).filter('name < ', query + u'\ufffd')
      search_type = 'Filename prefix search%s' % ['', ', matching case,'][match_case]
      #db.GqlQuery("SELECT * FROM MyModel WHERE prop >= :1 AND prop < :2", "abc", u"abc" + u"\ufffd")
  if results and limit:
    results.fetch(limit)
  return {'results': results, 'search_type': search_type, 'query': query, 'search_limit': limit, 'search_offset': offset}
Beispiel #9
0
def file_upload(user):
    
    _file = request.files['file']
    cont = ''
    for d in _file.stream:
        cont += d
    from binascii import hexlify
    cont = hexlify(cont)
    
    name = _file.filename
    root = models.get_dir(name=user, path='/')
    db_f = g.db.query(File).filter_by(owner=user, name=name, dir=root.inode).first()
    if db_f is not None:
        return "File already exists", 409

    new_file = File(name=name, owner=user, content=cont, dir=root.inode)
    new_file.directory = root
    tx = Transaction(user=user, 
                     action="CREATE", 
                     type="FILE", 
                     pathname=new_file.pathname(),
                     ip_address=request.remote_addr)
    g.db.add(new_file)
    g.db.add(tx)
    return "Success"
def download():
    '''Grabs the latest.'''

    # Download Files
    date_str = date.today().strftime("%Y_%m_%d")
    new_folder_path = "data/downloaded_%s" % date_str
    download_files(new_folder_path)


    # Delete if already exist in database
    for path, subdirs, files in os.walk(new_folder_path):
        for f in files:
            if already_downloaded(path + '/' + f):
                print "Didn't save '%s' because it was already in the database." % f
                os.remove(path + '/' + f)
            else:
                print "Saved new file '%s/%s'" % (path, f)

                File.create(
                    name = f,
                    years=next(re.finditer(r'\d{4}_\d{4}', f)),
                    sha1 = sha1OfFile(path + '/' + f),
                    updated = date.today(),
                    ingested = False
                )
Beispiel #11
0
def fstream(name):
    fstream_count = File.objects(name=name).count()
    if fstream_count:
        fstream = File.objects(name=name).first()
        return fstream.data.read()
    else:
        return ''
Beispiel #12
0
def add(request, key=None, type=FILE):
    to = key # lame but it does the trick for now
    if type == FOLDER:
        form = FolderForm(request.form)
    else:
        form = FileForm(request.form)
    if request.method == "POST" and form.validate():
        if len(form.slug.data) < 1:
            form.slug.data = slugify(form.name.data)
        if type == FOLDER:
            file = File.add(to=to,type=type, name=form.name.data,
                                            slug=form.slug.data,
                                            breadcrumb=form.breadcrumb.data,
                                            state=form.state.data,
                                            active=form.active.data,
                                            author=users.get_current_user(),
                                            updated=datetime.now())
        elif type == FILE:
            file = request.files.get('file')
            data = db.Blob(file.read())
            file = File.add(to=to,type=type, name=form.name.data,
                                            slug=form.slug.data,
                                            breadcrumb=form.breadcrumb.data,
                                            state=form.state.data,
                                            active=form.active.data,
                                            author=users.get_current_user(),
                                            updated=datetime.now(),
                                            content_type=file.content_type,
                                            data=data, size=len(data))

        if form.save.data is True:
            return redirect(url_for('nut:files/list'), 301)
        if form.cont.data is True:
            return redirect(url_for('nut:files/edit', key=file.key()), 301)
    return render_template('app:files/form.html', form=form)
Beispiel #13
0
 def test_model_file(self):
     """Test File Model"""
     folder = Folder(name='test')
     folder.save()
     obj = File(name='test', folder=folder)
     obj.save()
     self.assertEquals(folder, obj.folder)
     self.assertNotEquals(obj.id, None)
     obj.delete()
Beispiel #14
0
 def create_file(self, filename, user=None):
     file = File()
     if user is None:
         user = self.get_current_user()
     file.author = user
     file.name = filename
     key = file.put()
     self.create_permission(file, file.author, ACCESS_WRITE)
     #self.redirect('/file?' + urllib.urlencode({'id': key}))
     return key
def already_downloaded(filepath):
    '''
    Return true if we already have this version of the file
    (check date and file hash). False otherwise
    '''
    try:
        File.get(File.sha1 == sha1OfFile(filepath))
        return True
    except peewee.DoesNotExist:
        return False
Beispiel #16
0
def parse(query):
    if query == "":
        return File.select()

    tokens = tokenize(query)
    result = _parse(iter(tokens))
    try:
        iter(result)
    except TypeError:
        result = File.select().join(Metadata, peewee.JOIN_LEFT_OUTER).where(result)
    return result
Beispiel #17
0
def parse_pep8(run, git_path, output):
    """Parse the pep8 output, store the results"""

    errfiles_set = set()
    errortype_set = set()
    lineno_set = set()

    # Add all files in the project to the db
    allfiles = set()
    os.path.walk(git_path, add_file_to_set, allfiles)
    for filename in allfiles:
        filename = filename.replace(git_path + '/', '', 1)
        runfile = File(filename=filename, run=run)
        runfile.save()

    # Generate a set of error types, error files, and lines
    for line in output.readlines():
        filename, lineno, errnum, errtext = string.split(line, ':', 3)
        lineno = int(lineno)
        filename = filename.replace(git_path + '/', '', 1)

        # Create sets to remove duplicates
        errfiles_set.add(filename)

        # Add new err types to the db
        if (errnum, errtext) not in errortype_set:
            errortype_set.add((errnum, errtext))
            if not Error.objects.filter(error_type=errnum):
                err = Error(error_type=errnum, short_descr=errtext)
                err.save()

        # Create a set of line numbers for each file
        for ln in range(max(1, lineno - 3), lineno + 4):
            lineno_set.add((filename, ln))

        # Add err instances to the db
        runfile = File.objects.get(run=run, filename=filename)
        errtype = Error.objects.get(error_type=errnum)
        runerr = RunError(error=errtype, file=runfile, line_number=lineno,
                          error_descr=errtext)
        runerr.save()

    # Add lines to the db
    for filename in errfiles_set:
        runfile = File.objects.get(run=run, filename=filename)

        f = open(git_path + '/' + filename, 'r')
        lineno = 1
        for line in f:
            if (filename, lineno) in lineno_set:
                linetext = Line(file=runfile, line_number=lineno, text=line)
                linetext.save()
            lineno = lineno + 1
        f.close()
Beispiel #18
0
def summary():
    text = '<h1>Database Summary</h1>\n'
    file1 = File.get(id=1)
    file2 = File.get(id=2)
    file3 = File.get(id=3)

    text += '<h2>' + file1.name + '</h2>\n'
    text += '<h2>' + file2.name + '</h2>\n'
    text += '<h2>' + file3.name + '</h2>\n'

    return text
Beispiel #19
0
def save_file(mime, base64):
    '''
    Calculate hash of the file, look up in the database,
    save if it's a new one and return its hash.  The hash
    can the be used as URL.
    '''
    hash_ = file_keeper_hash(mime, base64)
    objects = File.objects.filter(hash=hash_)
    if not objects:
        file_ = File(hash=hash_, mime=mime, base64=base64)
        file_.save()
    return hash_
Beispiel #20
0
def get_entries():
    """Gets all file names currently in catalog"""

    catalog_files = []

    if File.select():
        for item in File.select():
            catalog_files.append(item.file_name)
    else:
        return catalog_files

    return catalog_files
Beispiel #21
0
def post_object(request):

    if request.method == "POST":

        upload = request.FILES['file']
        parent = request.POST.get('parent') if 'parent' in request.POST.keys() else None

        f = File(name=upload.name, file=upload, owner=request.user, parent=parent, bytes=request.POST.get('size'), mime=request.POST.get('type'))
        f.save()

        return JsonResponse(f.get_client_inode(), safe=False)
    return JsonResponse({'message':'Invalid file..'}, status=400, safe=False)
    def put(self, filename):
        file_uuid = uuid4().hex

        file = File(name=str(urllib.unquote(filename)),
             content_type=self.request.headers.get('Content-Type', None),
             key_name=file_uuid)
        file.put()

        for chunk in chunks(self.request.body, config.max_fragment_size):
            Fragment(file=file, data=chunk).put()

        self.response.set_status(201)
        self.response.out.write(file_uuid)
Beispiel #23
0
def order():
    if 'username' not in session:
        return redirect(url_for('login'))

    form = forms.OrderForm(formdata=request.form)
    form.category.choices = [(c.id, c.name) for c in Category.query.filter_by(subject_id=form.subject.data)]

    if ('0', 'Select...') not in form.subject.choices:
        form.subject.choices.insert(0, ('0', 'Select...'))

    if request.method == 'POST' and form.validate():

        new_order = Order()

        new_order.title = form.title.data
        new_order.user_id = session['user_id']
        new_order.subject_id = form.subject.data
        new_order.category_id = form.category.data
        new_order.os_id = form.os.data
        new_order.details = form.details.data
        new_order.deadline = form.deadline.data
        new_order.explanations = form.explanations.data

        db.session.add(new_order)
        db.session.commit()

        files = request.files.getlist('file[]')
        order_files = []
        for f in files:
            if f.filename:
                ext = f.filename.split('.')[-1]
                name = str(uuid.uuid4()) + '.' + ext
                path = os.path.join(app.config['FILE_UPLOAD_PATH'], name)
                f.save(path)

                of = File()
                of.order_id = new_order.id
                of.name = f.filename
                of.local_name = name
                order_files.append(of)

        for of in order_files:
            db.session.add(of)

        db.session.commit()

        flash('Order submitted :)', _CATEGORY_MESSAGE)

        return redirect(url_for('profile'))

    return render_template('order.html', form=form)
Beispiel #24
0
def fileRead(request):
    if request.method == 'POST':
        myfile = request.FILES['myfile']
        fs = FileSystemStorage()
        filename = fs.save(myfile.name, myfile)
        uploaded_file_url = fs.url(filename)
        ob = File(path="/home/divum/Desktop/django/myproject" +
                  uploaded_file_url)
        ob.save()
        dic = {}
        dic['status'] = 200
        dic['message'] = "File Stored"
        dic['url'] = "/home/divum/Desktop/django/myproject" + uploaded_file_url
        return JsonResponse(dic, status=200)
Beispiel #25
0
def save_file(uploaded_file, release_id):
    if uploaded_file: # and allowed_file(uploaded_file.filename):
        filename = secure_filename(uploaded_file.filename)
        if os.path.isfile(os.path.join(app.config['UPLOAD_FOLDER'], filename)):
            fileName, fileExtension = os.path.splitext(filename)
            filename = fileName + '-' + time.strftime("%Y%m%d-%H%M%S") + fileExtension
        uploaded_file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))

        description = "";
        statinfo = os.stat(os.path.join(app.config['UPLOAD_FOLDER'], filename))
        file = File(release_id=release_id, filename=filename, description=description, size=statinfo.st_size)
        file.approved = False;
        db.session.add(file)
        db.session.commit()
Beispiel #26
0
def edit_post(year, month, day, slug):
    try:
        p = Post.objects.get(site=g.site.domain,
                             year=year,
                             month=month,
                             day=day,
                             slug=slug)
    except Post.DoesNotExist:
        abort(404)

    if not g.site.domain == g.user:
        abort(403)

    if request.method == "POST":
        reqfile = request.files.get('file')
        if reqfile:
            f = File()
            f.site = g.site.domain
            f.name = reqfile.filename
            f.content_type = reqfile.mimetype
            f.slug, f.content_length = save_file(
                reqfile, current_app.config["UPLOAD_FOLDER"])
            f.save()

        p.name = request.form.get("name")
        p.text = request.form.get("text")
        if reqfile:
            p.image_slug = f.slug
        p.save()
        return redirect(
            url_for("post", year=p.year, month=p.month, day=p.day,
                    slug=p.slug))

    return render_template("edit_post.html", post=p)
Beispiel #27
0
	def __init__(self, request):
		self._R = Request(request)
		self._P = self._R.extractPatterns()
		self._fileName = self._R.extractFileName()
		self._F = File(self._fileName)
		self._matchFile, self._unmatchFile = self._F.getPostProcessingFileNames()


		self._M = None
		self._UM = None

		self._umCount = 0
		self._mCount = 0

		self._fileData = []
Beispiel #28
0
def uploads():
    if request.method == 'GET':
        return render_template('uploads.html')
    else:
        f = request.files['file']
        f.save(
            os.path.join(os.path.dirname('uploads'),
                         secure_filename(f.filename)))
        file = File(file_name=str(secure_filename(f.filename)))
        user_id = session.get('user_id')
        user = User.query.filter(User.id == user_id).first()
        file.author = user
        db.session.add(file)
        db.session.commit()
        return redirect(url_for('downloads'))
Beispiel #29
0
def upload_html(request, submission):
    """
    Helper function that processes the file upload.  Used by the 'no flash' version.
    
    :Return: Void
    """
    
    if request.FILES:
        for file in request.FILES:
            upload = File()
            upload.submission = submission
            upload.file_upload = request.FILES[file]
            upload.save()
            
    return
Beispiel #30
0
def add_file(request,prop_id, event_id, note_id):
	if request.method == 'POST':
		form = FileUploadForm(request.POST, request.FILES)
		note = Note.objects.get(pk=note_id)
		if form.is_valid():
			newdoc = File(docfile=request.FILES['docfile'] )
			newdoc.note = note
			newdoc.save()
			return HttpResponse("added file")
		else:
			form = FileUploadForm()
		documents = File.objects.all()
		context={'form':form, 'documents': documents,'event_id':event_id,
		'prop_id':prop_id,"note_id":note_id}
		return HttpResponseBadRequest(render (request,'main/note.html',context))
def upload(request):
    if request.POST:
        userid = request.POST.get('userid')
        
        # Queries all non-expired sessions
        sessions = Session.objects.filter(expire_date__gte=datetime.now())

        # Checks if session is active
        for session in sessions:
            data = session.get_decoded()
            found_userid=data.get('_auth_user_id')

            # Prceeds when user id is validated
            if found_userid!=None and long(userid)==found_userid:
                user = User.objects.filter(id=userid)[0]
                faculty=None
                faculty=Faculty.objects.filter(id=request.POST.get('fid'))[0]
                transaction = Transaction.objects.get(id=request.POST.get('transaction'))
                document = Dokument()
                document.faculty= faculty
                document.transaction= transaction
                document.save()

                #Generates a random alphanum string for filename template
                while True:
                    fnameTemplate=''
                    fnameTemplate = ''.join(random.choice(string.ascii_lowercase))
                    fnameTemplate += ''.join(random.choice(string.ascii_lowercase + string.digits) for x in range(4)) + '_'
                    if len(File.objects.filter(filename__startswith = fnameTemplate))==0: break

                #Processes uploaded files, page by page
                for key in request.FILES:
                    files = request.FILES[key]
                    filename = fnameTemplate + key.split('_')[1] + '.bmp'
                    with open('DCSArchivingSystem/testapp/media/files/' + filename, 'wb+') as destination:
                        for chunk in files.chunks():
                            destination.write(chunk)
                        file = File()
                        file.filename = filename
                        file.file = 'files/' + filename
                        file.save()                    
                        document.files.add(file)
                    Log.create(user, "Uploaded file", file, document).save()    
                Log.create(user, "Created Document", None, document).save()    
                return HttpResponseRedirect("/dashboard/")
            
    else:
        return render_to_response('upload.html', context_instance=RequestContext(request))
Beispiel #32
0
    async def post(self):
        with self.make_session() as session:
            self._session = session
            args, user = await self.init_API_request()

            if user:
                url = args.get("file_URL")
                a = urlparse(url)
                filename = os.path.join("uploads", os.path.basename(a.path))

                filename, headers = await tornado.ioloop.IOLoop.current(
                ).run_in_executor(
                    None,
                    urllib.request.urlretrieve,
                    url,
                    filename,
                )

                if filename:
                    file = File(user.id, 0, filename)
                    session.add(file)
                    session.commit()

                    json_response = {"filename": filename, "file_id": file.id}

                    self.write(json_response)
                else:
                    self.write({"error": "Failed to download file"})
Beispiel #33
0
def add_file_to_database(submitter, title, original_name, description,
                         file_content, file_mime, file_hash):
    """
    This:
    1) Writes a file to the disk ({UPLOAD_FOLDER}/hash),
    2) Attempts to generate a transcript of the file,
    3) Saves the file to the database.
    """
    # save file to UPLOAD_FOLDER/file_hash
    save_location = ''.join(
        [current_app.config['UPLOAD_FOLDER'], "/", file_hash])
    with open(save_location, "wb+") as storage:
        storage.write(file_content)  # actually write the file

    try:
        text = textract.process(
            save_location,
            language='eng',
            extension=get_extension_from_mimetype(file_mime)).decode('utf-8')
    except (textract.exceptions.UnknownMethod, textract.exceptions.ShellError):
        text = ""

    new_file = File(title=title,
                    original_name=original_name,
                    description=description,
                    file_path=save_location,
                    file_mime=file_mime,
                    transcript=text,
                    file_hash=file_hash,
                    submitter=submitter)

    APP_DATABASE.session.add(new_file)  # add to database
    APP_DATABASE.session.commit()
    return new_file.id
Beispiel #34
0
def file_factory(**kwargs):
    base_parameters = dict(bucket_path='/test-job/my-foto.png',
                           created_at=datetime.datetime(2019, 3, 1))
    file = File(**dict(**kwargs, **base_parameters))
    db.session.add(file)
    db.session.commit()
    return file
Beispiel #35
0
def add_files(request, nonce=''):
    submission = get_object_or_404(UserSubmission, nonce=nonce)

    # don't let you edit the object if it's processing or processed.
    # this is CRITICAL, be very careful modifying this stanza or the above!
    if submission.is_processing():
        return HttpResponseRedirect(reverse(process, kwargs={'nonce': submission.nonce}))
    elif not submission.can_process():
        return HttpResponseRedirect(reverse(review, kwargs={'nonce': submission.nonce}))

    files = File.objects.filter(usersubmission=submission)
    fileinstance = File(usersubmission=submission)

    if request.method == 'POST':
        form = FileForm(request.POST, request.FILES, instance=fileinstance)
        if form.is_valid():
           uploaded_file_instance = form.save(commit=False) 
           uploaded_file_instance.filename = request.FILES['path'].name
           uploaded_file_instance.save()
           return HttpResponseRedirect(submission.get_absolute_url())
    else:
        form = FileForm(instance = fileinstance)

    return render_to_response('bookmaker/add_files.html', 
                                { 'form': form, 'submission': submission, 'files': files },
                                context_instance=RequestContext(request))
Beispiel #36
0
def get_data(text, lang='en'):
    if lang != 'en':
        return None

    url = f'https://howjsay.com/mp3/{quote(text)}.mp3'
    if utils.url_exists(url):
        yield ('audio', File(url=url, region=None))
Beispiel #37
0
def ingest(filepath):
    '''Ingest file into database'''

    print "Ingesting %s" % filepath
    rows_in_file = parse_fec_file(filepath)
    myfile = File.get_or_create(name=filepath)
    myfile_id = myfile.id

    with db.transaction(): # TODO: More sane error handling
        for idx in range(0, len(rows_in_file), 500):

            # Ingest 500 rows at a time
            print "Inserting row %d of %s" % (idx, filepath)
            rows_subset = rows_in_file[idx:idx+500]
            rows_to_insert = []

            for row in rows_subset:
                unsaved_new_contribution = Contribution(**row_to_dict(row))
                import pdb; pdb.set_trace()
                # If the row isn't already there, insert it
                if :
                    pass
                # If the row is there, check for modifications
                elif:
                    # If it has not been modified, simply add a ContributionHistory object
                    if:
                    # If it has been modified, create a new object and give the new object a contribution history
                    else:
                        pass

            Contribution.insert_many(rows_subset).execute()
Beispiel #38
0
def get_data(text, lang='ru'):
    pat = 'https://ru.forvo.com/word/{0}/#{1}'
    url = pat.format(urllib.parse.quote(text), lang)
    headers = {
        'User-Agent': utils.CHROME_USER_AGENT,
        'Accept': 'text/html',
    }
    resp = requests.get(url, headers=headers)
    resp.raise_for_status()

    soup = BeautifulSoup(resp.text, 'html.parser')
    article = soup.find('article', class_='pronunciations')
    if article is None:
        return None
    ul = article.find('ul', class_="show-all-pronunciations")
    if ul is None:
        return None

    li = ul.find_all('li')
    parsed_items = [parse_item(t) for t in li]
    items = [
        t for t in parsed_items if t is not None and utils.url_exists(t['url'])
    ]

    data = {'audio': []}
    for item in items:
        data['audio'].append(File(url=item['url'], region=None))

    return data
Beispiel #39
0
def file_upload():
    if request.method == 'POST':
        logger.info(request.form)
        form = FileUploadForm(CombinedMultiDict((request.files, request.form)))
        file = request.files.get('file')
        form.hash.data = hasher.hash(file)
        logger.info(form.hash.data)
        if form.validate():
            logger.info('Form is valid')
            file.stream.seek(0)
            timestamp = int(datetime.datetime.now().timestamp())
            filename = str(timestamp) + secure_filename(file.filename)
            file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
            logger.info(filename + ' Saved')
            db_file = File(name=form.upload_name.data,
                           cve_number=form.cve_number.data,
                           full_path=os.path.join(app.config['UPLOAD_FOLDER'],
                                                  filename),
                           description=form.description.data,
                           file_type=form.upload_type.data,
                           hash=form.hash.data)
            db.session.add(db_file)
            db.session.commit()
            # logger.info(str(db_file) + ' saved to db')
            flash('File Uploaded', 'success')
        else:
            # logger.info(form.errors)
            flash_form_errors(form)
        if request.referrer:
            return redirect(request.referrer)
        else:
            return redirect(url_for('index'))
Beispiel #40
0
def upload_file(path):
    if request.method == "POST":
        file = request.files['file']

        filename = ".".join(secure_filename(
            file.filename).split(".")[:-1]) + "_" + str(current_user.id)
        ext = os.path.splitext(file.filename)[-1]
        filename = filename + ext
        file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
        file.save(file_path)
        parent_path = request.form.get('parent_path')
        size = os.path.getsize(file_path)
        filedata = File(user_id=current_user.id,
                        child_path=filename,
                        type=ext,
                        original_path="/home/ubuntu/data",
                        filename=filename,
                        size=size,
                        parent_path='/' + parent_path.strip('/') + '/')
        db.session.add(filedata)
        db.session.commit()
        return redirect(url_for("data", path=path))

    else:
        return render_template("upload_file.html", path=path)
Beispiel #41
0
    def upload(self):
        celery = create_celery(current_app)
        if request.method == 'POST':
            if 'folder' in request.files:
                files = request.files.getlist('folder')

                for f in files:
                    filename = secure_filename(f.filename)[3:]
                    symbol, short_name, contract_date = _parse_contract_date(
                        filename)
                    s3.put_object(Bucket=os.environ.get('S3_BUCKET_NAME'),
                                  Key='downloads2/{}'.format(f.filename),
                                  Body=f)
                    url = 'https://s3.amazonaws.com/{}/downloads2/{}/{}'.format(
                        os.environ.get('S3_BUCKET_NAME'), symbol, short_name)
                    db.session.add(
                        File(filename=filename,
                             path=url,
                             symbol=symbol,
                             contract_date=contract_date))
                db.session.commit()
                # celery.send_task('tasks.write_filetable', args=(filename, url, symbol, contract_date))
                flash("Your files have been successfully uploaded!", "success")
                return redirect(request.url)
            return redirect(request.url)
        return self.render('admin/upload_file.html')
Beispiel #42
0
def Token(token):
    if ':' in token:
        field, value = token.split(':', 1)
    else:
        field, value = 'tag', token
    files = File.select().join(Metadata, peewee.JOIN_LEFT_OUTER)
    return set(files.where((Metadata.field == field) & (Metadata.value == value)))
Beispiel #43
0
    def setUp(self):
        self.group, created = Group.objects.get_or_create(name='test')
        self.user, created = DjangoUser.objects.get_or_create(username=self.username)
        self.user.set_password(self.password)
        self.user.save()
        perspective, created = Perspective.objects.get_or_create(
            name='default')
        perspective.set_default_user()
        perspective.save()

        ModuleSetting.set('default_perspective', perspective.id)

        self.folder = Folder(name='test')
        self.folder.set_default_user()
        self.folder.save()

        self.document = Document(title='test_document', folder=self.folder)
        self.document.set_default_user()
        self.document.save()

        self.file = File(name='test_file', folder=self.folder)
        self.file.set_default_user()
        self.file.save()

        self.link = WebLink(title='test', folder=self.folder, url='test')
        self.link.set_default_user()
        self.link.save()
Beispiel #44
0
def upload():
    if not request.files['file'].filename:
        flash(u'请上传文件')
        return redirect(url_for('index'))

    f = request.files['file']
    files = [file.name for file in File.query.all()]
    if f.filename in files:
        flash(u'文件已存在, 直接下载')
        return redirect(url_for('index'))

    # 将文件存至服务器
    dirname = os.path.join(os.path.dirname(__file__), 'files')
    try:
        f.save(os.path.join(dirname, f.filename))
    except Exception as e:
        logger.error(e)
        return redirect(url_for('index'))
    # 将信息存入数据库
    file = File(name=f.filename, size=request.content_length)
    try:
        db.session.add(file)
        db.session.commit()
    except Exception as e:
        logger.error(e)
        return redirect(url_for('index'))

    return redirect(url_for('index'))
Beispiel #45
0
def upload():
    file = request.files['file']

    try:
        days = float(request.form['days'])
    except ValueError:
        days = 0
    try:
        hrs = float(request.form['hours'])
    except ValueError:
        hrs = 0

    if file.filename == '' or (days == 0 and hrs == 0):
        flash("Incorrect file data!")
    else:
        expire_at = datetime.datetime.now()
        expire_at += datetime.timedelta(days=days, hours=hrs)
        expire_at = expire_at.replace(microsecond=0)

        new_file = File(file.filename, file.read(), expire_at,
                        None if current_user.is_anonymous else current_user.id)

        db.session.add(new_file)
        db.session.commit()
        flash(f"File {file.filename} will be stored until {expire_at}!\n"
              f"The file can be accessed at "
              f"{url_for('file', file_id=new_file.id, _external=True)}!")

    return jsonify(message="OK")
Beispiel #46
0
def download(file_hash):
    paste_file = File.get_by_filehash(filehash=file_hash)
    return send_file(open(paste_file.path, 'rb'),
                     mimetype='application/octet-stream',
                     cache_timeout=ONE_MONTH,
                     as_attachment=True,
                     attachment_filename=paste_file.filename.encode('utf-8'))
Beispiel #47
0
def process(args, pool, log):
    files = File.select().where(File.inspected_at.is_null()).execute()
    paths = [pathlib.Path(f.path) for f in files]

    count_total = file_count_total(paths)
    size_total = file_size_total(paths)

    parse = functools.partial(record_for, args)

    count_processed = 0
    size_processed = 0

    for (path, result, messages) in pool.imap(parse, paths):
        #for (path, result, messages) in map(parse, input_iterator(args)):
        count_processed += 1
        size_processed += file_size(path)

        for message in messages:
            print('error: {}: {}'.format(message, path))

        if result is None:
            mark_processed(path)
            continue

        save_media(result)

        print('process: {} {} {}/{} {}/{}'.format(
            result['source'], result['path'], count_processed, count_total,
            humanize.naturalsize(size_processed),
            humanize.naturalsize(size_total)))
Beispiel #48
0
    def post(self):

        user = users.get_current_user()
        if user == None:
            self.redirect('/')
        else:

            current_user = MyUser.get(user.user_id())

            dir_id = self.request.get('dir')

            owner_user, directory, parent = check(current_user.key, dir_id)

            if owner_user and directory and (
                    directory.is_users_dir(current_user.key)
                    or has_write_permission(current_user.key, parent)):

                for upload in self.get_uploads():

                    blobinfo = blobstore.BlobInfo(upload.key())

                    my_file = File(name=blobinfo.filename, blob=upload.key())

                    if not directory.file_exists(my_file.name):
                        directory.files.append(my_file)

                directory.put()

                self.redirect(self.request.referer)
Beispiel #49
0
def share(path):
    is_public = False

    try:
        f = File.get(File.public_share_url == path)
        actual_filename = generate_filename(f.folder.name, f.filename)
        target_file = os.path.join(os.path.expanduser(config.UPLOAD_FOLDER),
                                   actual_filename)
        is_public = True
    except peewee.DoesNotExist:
        return jsonify(message='error'), 404

    if not (is_public and f.open_public_share):
        return jsonify(message='error'), 404

    s = URLSafeSerializer(config.SECRET_KEY, expires_in=24 * 3600)

    args = request.args
    if args.get('download') == 'true':
        return redirect("/share/download/" + path + "/" + f.filename)

    share_token = s.dumps({'path': path}).decode('utf-8')

    payload = {
        'filename': f.filename,
        'folder': f.folder.name,
        'open_public_share': f.open_public_share,
        'share_token': share_token,
    }

    return jsonify(message='OK', data=payload)
Beispiel #50
0
    def update_metadata(self):
        metadata = File.get(md5=self.preview).get_metadata()
        # Handle Static markup
        markup = ""
        markup += "<b>Name:</b> %s<br />" % metadata['name']
        markup += "<b>Path:</b> %s<br />" % metadata['path']
        markup += "<b>md5:</b> %s<br />" % metadata['md5']
        markup += "<b>Width:</b> %s<br />" % metadata['width']
        markup += "<b>Height:</b> %s<br />" % metadata['height']
        markup += "<b>Type:</b> %s<br />" % metadata['type']
        markup += "<b>size:</b> %s<br />" % metadata['size']

        # Handle dynamic Markup
        fields = defaultdict(set)
        for m in metadata['metadata']:
            fields[m.field].add(m.value)

        for field, values in fields.items():
            links = [
                "<a href='%s:\"%s\"'>%s</a>" % (field, value, value)
                for value in values
            ]
            markup += "<b>%s:</b> %s<br />" % (field, ", ".join(links))

        self.DetailsWindow.SetPage(markup)
Beispiel #51
0
    def create_instance(self, body_data):
        """
        重载修改后:
        *.将password字段加密

        :param body_data: 创建对象时的传入数据
        :return: instance: 创建操作后的对象
        """
        password = generate_password_hash(body_data["password"],
                                          method="pbkdf2:sha256")
        body_data["password"] = password
        body_data["creator_id"] = g.login_user.id
        if "img_url" not in body_data:
            body_data[
                "img_url"] = "https://gw.alipayobjects.com/zos/rmsportal/BiazfanxmamNRoxxVxka.png"
        # 判断该模型是否支持所有输入属性
        for item in body_data:
            if not hasattr(self.model, item):
                del body_data[item]
        # 创建对象
        user = self.model(**body_data)
        db.session.add(user)
        db.session.commit()

        # 创建空间
        space_data = {
            "name": user.username + "'s private space",
            "space_type": "private",
            "own_user_id": user.id
        }
        space = Space(**space_data)
        db.session.add(space)
        db.session.commit()

        # 创建桶
        bucket_data = {
            "name": space.name + "'s 1st bucket",
            "space_id": space.id
        }
        bucket = Bucket(**bucket_data)
        db.session.add(bucket)
        db.session.commit()

        # 创建空间根目录
        bucket_data = {
            "object_name": space.name + "'s root",
            "object_type": "folder",
            "object_size": 0,
            "creator_id": user.id,
            "bucket_id": bucket.id
        }
        folder_root = File(**bucket_data)
        db.session.add(folder_root)
        db.session.commit()

        # 关联空间与根目录
        space.root_folder = folder_root
        db.session.commit()

        return user
Beispiel #52
0
def share_download(path, filename):
    try:
        f = File.get(File.public_share_url == path)
        actual_filename = generate_filename(f.folder.name, f.filename)
        target_file = os.path.join(os.path.expanduser(config.UPLOAD_FOLDER),
                                   actual_filename)
    except peewee.DoesNotExist:
        return jsonify(message='error'), 404

    s = URLSafeSerializer(config.SECRET_KEY, expires_in=24 * 3600)
    share_token = None
    cookies = request.cookies
    if 'share_token' in cookies:
        share_token = cookies['share_token']
        try:
            data = s.loads(share_token)
            if data['path'] == path:
                if os.path.exists(target_file):
                    return send_file(target_file)
                else:
                    return jsonify(message='error'), 404
            else:
                return jsonify(message='unauthorized'), 401
        except:
            return jsonify(message='unauthorized'), 401
Beispiel #53
0
def file_post():
    file = request.files.get("file")
    if not file:
        data = {"message": "Could not find file data"}
        return Response(json.dumps(data), 422, mimetype="application/json")

    filename = secure_filename(file.filename)
    print "printing upload path:"
    print upload_path(filename)
    db_file = File(name=filename)
    session.add(db_file)
    session.commit()
    file.save(upload_path(filename))                                                                          
    data = db_file.as_dictionary()
    print data
    return Response(json.dumps(data), 201, mimetype="application/json")
Beispiel #54
0
def view_all_entries():
    """Gather all entries in the catalog, along with their metadata"""

    all_entries = (FileTag
                   .select(FileTag, File, Tag)
                   .join(Tag)
                   .switch(FileTag)
                   .join(File)
                   .order_by(File.file_name))

    # dictionary houses all files in catalog and each file's associated tags
    files_with_tags = {}

    for file_tag in all_entries:

        f = file_tag.file_id
        tag = file_tag.tag_id.tag_name

        files_with_tags.setdefault(f.file_name, []).append(tag)

    ordered_list = sorted([file_name for file_name in File.select()])

    # Creates a list called of tuples called 'entries' 
    # each tuple containing name of catalog file and metadata
    entries = []
    for f_name in ordered_list:
        f_name.file_name
        f_name.description
        f_name.date_created
        tags = ' | '.join(sorted(files_with_tags.get(f_name.file_name, '')))

        entries.append((f_name.file_name, f_name.description, f_name.date_created, tags))

    return entries
def upload_file(f):
    print(f.filename, file=sys.stdout)
    filename = secure_filename(f.filename)
    fullpath = os.path.join(app.config['UPLOAD_FOLDER'], filename)
    f.save(fullpath)
    newFile = File(name=filename, path=fullpath)
    db_session.add(newFile)
    db_session.commit()
Beispiel #56
0
def folder(folder_name):
    try:
        folder = Folder.get(Folder.name == folder_name)
    except peewee.DoesNotExist:
        return jsonify(message="error"), 404

    if request.method == "GET":
        return jsonify(message="OK", data=model_to_dict(folder, backrefs=True))

    if request.method == "POST":
        f = request.files["file"]
        if f:
            actual_filename = generate_filename(folder_name, f.filename)
            target_file = os.path.join(
                os.path.expanduser(config.UPLOAD_FOLDER), actual_filename)
            if os.path.exists(target_file):
                return jsonify(message="error"), 409

            try:
                f.save(target_file)
                f2 = File.create(folder=folder,
                                 filename=f.filename,
                                 public_share_url=generate_url(),
                                 open_public_share=False)
                f2.save()
            except Exception as e:
                print(e)
                return jsonify(message="error"), 500

            return jsonify(message="OK"), 201

    if request.method == "DELETE":
        try:
            for f in folder.files:
                actual_filename = generate_filename(folder_name, f.filename)
                target_file = os.path.join(
                    os.path.expanduser(config.UPLOAD_FOLDER), actual_filename)
                f2 = File.get(File.filename == f.filename)
                f2.delete_instance()
                if os.path.exists(target_file):
                    os.remove(target_file)
            folder.delete_instance()
        except Exception as e:
            return jsonify(message="error"), 409

    return jsonify(message="OK")
Beispiel #57
0
    def get(self):
        obj = {
            'categories': Category.select()[:],
            'items': Item.select()[:],
            'files': File.select()[:]
        }

        return json_response(obj, exclude=('blob'))
Beispiel #58
0
def file_post():
    print(request.files)
    if 'file' not in request.files:
        flash('no file part')
        return redirect('/')
    file = request.files['file']
    if file.filename == '':
        flash('not selected file')
        return redirect('/')
    filename = secure_filename(file.filename)
    file_path = upload_path(filename)
    file.save(file_path)
    file_sql = File(file_name=filename, path=file_path)
    session.add(file_sql)
    session.commit()
    jdump = jsonify(file_sql.as_dictionary())
    return jdump, 201
Beispiel #59
0
def add_files(text_set_id):
    text_set_id = int(text_set_id)
    text_set = text_set_logic.get(text_set_id)

    uploaded_files = []
    for filename, file_stream in request.files.items():
        fileindex = re.search("file\\[(?P<index>\d+)\\]", filename).group('index')
        text_column = request.form.get("text_column[{}]".format(fileindex), '')
        text_column = int(text_column) if text_column else 1
        separator = request.form.get("separator[{}]".format(fileindex), '')
        separator = separator if separator else ','
        file = File()
        file.separator = separator
        file.text_column = text_column
        uploaded_files.append(file_logic.save(file, file_stream))
    text_set_logic.add_files_to_text_set(text_set_id, [file.id for file in uploaded_files])
    return jsonify(text_set.to_dict(_hide=[]))
Beispiel #60
0
def train():
    model_id = int(request.form["model_id"])
    model = db.session.query(EmbeddingModel).get(
        model_id)  # type: EmbeddingModel
    if model is None:
        raise ValueError("No embedding model found by id {0}".format(model_id))

    separator = request.form["separator"]
    text_column = int(request.form["text_column"])
    tag_column = int(request.form["tag_column"])
    positive_tag = request.form["positive_tag"]
    negative_tag = request.form["negative_tag"]
    uploaded_files = []
    for filename, file_stream in request.files.items():
        file = File()
        file.separator = separator
        file.text_column = text_column
        file.tag_column = tag_column
        file.positive_tag = positive_tag
        file.negative_tag = negative_tag
        uploaded_files.append(file_logic.save(file, file_stream))

    extend_vocabulary = True if request.form[
        "extend_vocabulary"] == "true" else False

    model = emb_model_logic.init_training(model_id,
                                          [file.id for file in uploaded_files],
                                          extend_vocabulary)

    return jsonify(model.to_dict())