Example #1
0
def _raise_if_locked(filepath): #Exception 401 if file is locked and lockid doesn't match as per the request

    i = web.input()

    host, port = utils.get_host_port(_config['lockserver'])
    if utils.is_locked(filepath, host, port, i.get('lock_id', None)):
        raise web.unauthorized()
Example #2
0
def index(request):
    if not is_authenticated(request):
        return HttpResponseRedirect(reverse('login'))

    if is_locked(request):
        return locked(request)

    return {
        'form': UploadFileForm()
    }
Example #3
0
def _raise_if_locked(filepath):
    """Raise a 401 unauthorized it the filepath is locked, and the
       appropriate locked wasn't given in the request.
    """

    i = web.input()

    host, port = utils.get_host_port(_config['lockserver'])
    if utils.is_locked(filepath, host, port, i.get('lock_id', None)):
        raise web.unauthorized()
Example #4
0
def _raise_if_locked(filepath):
    """Raise a 401 unauthorized it the filepath is locked, and the
       appropriate locked wasn't given in the request.
    """

    i = web.input()

    host, port = utils.get_host_port(_config['lockserver'])
    if utils.is_locked(filepath, host, port, i.get('lock_id', None)):
        raise web.unauthorized()
    def __init__(self, filepath, mode='rtc'):
        print("FilePath", filepath)
        print("FilePath", mode)
        #self.mode = mode
        #self.filepath = filepath

        host, port = utils.get_host_port(_config['nameserver'])
        self.srv = utils.get_server(filepath, host, port)
        print(self.srv)

        print("FilePath", filepath)
        print("FilePath", mode)

        if self.srv is None:
            raise DFSIOError('Could not find a server that serves :', filepath)

        self.last_modified = None
        SpooledTemporaryFile.__init__(
            self, _config['max_size'],
            mode.replace('c', ''))  #mode 'c' meaning store in cache

        host, port = utils.get_host_port(_config['lockserver'])
        if utils.is_locked(filepath, host, port):
            raise DFSIOError('The file %s is locked.' % filepath)

        if 'w' not in mode:
            print(type(self.srv))
            host, port = utils.get_host_port(self.srv.decode('UTF-8'))
            with closing(HTTPConnection(host, port)) as con:
                con.request('GET', filepath)
                response = con.getresponse()
                self.last_modified = response.getheader('Last-Modified')
                status = response.status
                print(status)

                if status not in (200, 204):
                    raise DFSIOError('Error (%d) while opening file.' % status)

                if status != 204:
                    self.write(response.read())

                if 'r' in mode:
                    self.seek(0)

                self.lock_id = None

        if 'a' in mode or 'w' in mode:
            # automatically gets a lock if we're in write/append mode
            host, port = utils.get_host_port(_config['lockserver'])
            self.lock_id = int(utils.get_lock(filepath, host, port))

        if 'c' in mode:
            File._cache[filepath] = self
Example #6
0
    def __init__(self, filepath, mode='rtc'):
        """filepath: the path of the distant file
           mode: take the same argument as mode argument of the global
                 open() + optional flag c (which mean store in cache).
        """

        self.mode = mode
        self.filepath = filepath
        host, port = utils.get_host_port(_config['nameserver'])
        self.srv = utils.get_server(filepath, host, port)

        if self.srv is None:
            raise DFSIOError('Impossible to find a server that serve %s.' %
                             filepath)

        self.last_modified = None
        SpooledTemporaryFile.__init__(self, _config['max_size'],
                                      mode.replace('c', ''))

        host, port = utils.get_host_port(_config['lockserver'])
        if utils.is_locked(filepath, host, port):
            raise DFSIOError('The file %s is locked.' % filepath)

        if 'w' not in mode:
            host, port = utils.get_host_port(self.srv)
            with closing(HTTPConnection(host, port)) as con:
                con.request('GET', filepath)
                response = con.getresponse()
                self.last_modified = response.getheader('Last-Modified')
                status = response.status

                if status not in (200, 204):
                    raise DFSIOError('Error (%d) while opening file.' % status)

                if status != 204:
                    self.write(response.read())

                if 'r' in mode:
                    self.seek(0)

                self.lock_id = None

        if 'a' in mode or 'w' in mode:
            # automatically gets a lock if we're in write/append mode
            host, port = utils.get_host_port(_config['lockserver'])
            self.lock_id = int(utils.get_lock(filepath, host, port))

        if 'c' in mode:
            File._cache[filepath] = self
Example #7
0
    def __init__(self, filepath, mode='rtc'):
        """filepath: the path of the distant file
           mode: take the same argument as mode argument of the global
                 open() + optional flag c (which mean store in cache).
        """

        self.mode = mode
        self.filepath = filepath
        host, port = utils.get_host_port(_config['nameserver'])
        self.srv = utils.get_server(filepath, host, port)

        if self.srv is None:
            raise DFSIOError('Impossible to find a server that serve %s.'
                    % filepath)

        self.last_modified = None
        SpooledTemporaryFile.__init__(self, _config['max_size'], mode.replace('c', ''))

        host, port = utils.get_host_port(_config['lockserver'])
        if utils.is_locked(filepath, host, port):
            raise DFSIOError('The file %s is locked.' % filepath)

        if 'w' not in mode:
            host, port = utils.get_host_port(self.srv)
            with closing(HTTPConnection(host, port)) as con:
                con.request('GET', filepath)
                response = con.getresponse()
                self.last_modified = response.getheader('Last-Modified')
                status = response.status

                if status not in (200, 204):
                    raise DFSIOError('Error (%d) while opening file.' % status)

                if status != 204:
                    self.write(response.read())

                if 'r' in mode:
                    self.seek(0)

                self.lock_id = None

        if 'a' in mode or 'w' in mode:
            # automatically gets a lock if we're in write/append mode
            host, port = utils.get_host_port(_config['lockserver'])
            self.lock_id = int(utils.get_lock(filepath, host, port))

        if 'c' in mode:
            File._cache[filepath] = self
Example #8
0
    def __init__(self, filepath, mode='rtc'):
        

        self.mode = mode
        self.filepath = filepath
        host, port = utils.get_host_port(_config['nameserver'])
        self.srv = utils.get_server(filepath, host, port)
        
        
        self.last_modified = None
        
        SpooledTemporaryFile.__init__(self, _config['max_size'], mode.replace('c',''))

        host, port = utils.get_host_port(_config['lockserver'])
        if utils.is_locked(filepath, host, port):
            raise DFSIOError('The file %s is locked.' % filepath)

        if 'w' not in mode:
            host, port = utils.get_host_port(self.srv)
            with closing(HTTPConnection(host, port)) as con:
                con.request('GET', filepath)
                response = con.getresponse()
                self.last_modified = response.getheader('Last-Modified')
                status = response.status

                if status not in (200, 204):
                    raise DFSIOError('Error (%d)while opening file.' % status)

                if status != 204:
                    self.write(response.read())

                if 'r' in mode:
                    self.seek(0)

                self.lock_id = None

        if 'a' in mode or 'w' in mode: # automatically lock file if appending or writing in file
            host, port = utils.get_host_port(_config['lockserver'])
            self.lock_id = int(utils.get_lock(filepath, host, port))

        if 'c' in mode:
            File._cache[filepath] = self        
    def __init__(self, path, mode='rtc'):

        self.mode = mode
        self.path = path
        host, port = utils.get_host_port(_config['nameserver'])
        self.srv = utils.get_server(path, host, port)

        if self.srv is None:
            pass

        self.last_modified = None
        SpooledTemporaryFile.__init__(self, _config['max_size'],
                                      mode.replace('c', ''))

        host, port = utils.get_host_port(_config['lockserver'])
        if utils.is_locked(path, host, port):
            pass

        if 'w' not in mode:
            host, port = utils.get_host_port(self.srv)
            with closing(HTTPConnection(host, port)) as con:
                con.request('GET', path)
                response = con.getresponse()
                self.last_modified = response.getheader('Last-Modified')
                status = response.status

                if status != 204:
                    self.write(response.read())

                if 'r' in mode:
                    self.seek(0)

                self.lock_id = None

        if 'a' in mode or 'w' in mode:
            host, port = utils.get_host_port(_config['lockserver'])
            self.lock_id = int(utils.get_lock(path, host, port))

        if 'c' in mode:
            File._cache[path] = self
Example #10
0
def upload(request):

    if is_locked(request):
        return HttpResponseRedirect(reverse('index'))

    file = request.FILES['file'].readlines()
    file_csv = csv.reader(file)
    urls_to_add = []

    for row in file_csv:
        url = row[0]
        try:
            validate(url) 
        except Exception:
            continue

        # If we are here, we have a valid url
        try:
            article_type = row[3].strip()
        except Exception:
            # Could be an index error, could not None
            article_type = ''

        urls_to_add.append({
            'url': url,
            'archive': True if article_type == 'Archive' else False
        })


    add_urls.delay(
        urls_to_add,
        request.session['oauth_token'],
        request.session['oauth_secret'])

    lock(request.session['oauth_token'])

    request.session['len_urls'] = len(urls_to_add)

    return HttpResponseRedirect(reverse('done'))