def _raise_if_locked(filepath): #Exception 401 if file is locked and lockid doesn't match as per the request
i = web.input()
host, port = utils.get_host_port(_config['lockserver'])
if utils.is_locked(filepath, host, port, i.get('lock_id', None)):
raise web.unauthorized()
def index(request):
if not is_authenticated(request):
return HttpResponseRedirect(reverse('login'))
if is_locked(request):
return locked(request)
return {
'form': UploadFileForm()
}
def _raise_if_locked(filepath):
"""Raise a 401 unauthorized it the filepath is locked, and the
appropriate locked wasn't given in the request.
"""
i = web.input()
host, port = utils.get_host_port(_config['lockserver'])
if utils.is_locked(filepath, host, port, i.get('lock_id', None)):
raise web.unauthorized()
def _raise_if_locked(filepath):
"""Raise a 401 unauthorized it the filepath is locked, and the
appropriate locked wasn't given in the request.
"""
i = web.input()
host, port = utils.get_host_port(_config['lockserver'])
if utils.is_locked(filepath, host, port, i.get('lock_id', None)):
raise web.unauthorized()
def __init__(self, filepath, mode='rtc'):
print("FilePath", filepath)
print("FilePath", mode)
#self.mode = mode
#self.filepath = filepath
host, port = utils.get_host_port(_config['nameserver'])
self.srv = utils.get_server(filepath, host, port)
print(self.srv)
print("FilePath", filepath)
print("FilePath", mode)
if self.srv is None:
raise DFSIOError('Could not find a server that serves :', filepath)
self.last_modified = None
SpooledTemporaryFile.__init__(
self, _config['max_size'],
mode.replace('c', '')) #mode 'c' meaning store in cache
host, port = utils.get_host_port(_config['lockserver'])
if utils.is_locked(filepath, host, port):
raise DFSIOError('The file %s is locked.' % filepath)
if 'w' not in mode:
print(type(self.srv))
host, port = utils.get_host_port(self.srv.decode('UTF-8'))
with closing(HTTPConnection(host, port)) as con:
con.request('GET', filepath)
response = con.getresponse()
self.last_modified = response.getheader('Last-Modified')
status = response.status
print(status)
if status not in (200, 204):
raise DFSIOError('Error (%d) while opening file.' % status)
if status != 204:
self.write(response.read())
if 'r' in mode:
self.seek(0)
self.lock_id = None
if 'a' in mode or 'w' in mode:
# automatically gets a lock if we're in write/append mode
host, port = utils.get_host_port(_config['lockserver'])
self.lock_id = int(utils.get_lock(filepath, host, port))
if 'c' in mode:
File._cache[filepath] = self
def __init__(self, filepath, mode='rtc'):
"""filepath: the path of the distant file
mode: take the same argument as mode argument of the global
open() + optional flag c (which mean store in cache).
"""
self.mode = mode
self.filepath = filepath
host, port = utils.get_host_port(_config['nameserver'])
self.srv = utils.get_server(filepath, host, port)
if self.srv is None:
raise DFSIOError('Impossible to find a server that serve %s.' %
filepath)
self.last_modified = None
SpooledTemporaryFile.__init__(self, _config['max_size'],
mode.replace('c', ''))
host, port = utils.get_host_port(_config['lockserver'])
if utils.is_locked(filepath, host, port):
raise DFSIOError('The file %s is locked.' % filepath)
if 'w' not in mode:
host, port = utils.get_host_port(self.srv)
with closing(HTTPConnection(host, port)) as con:
con.request('GET', filepath)
response = con.getresponse()
self.last_modified = response.getheader('Last-Modified')
status = response.status
if status not in (200, 204):
raise DFSIOError('Error (%d) while opening file.' % status)
if status != 204:
self.write(response.read())
if 'r' in mode:
self.seek(0)
self.lock_id = None
if 'a' in mode or 'w' in mode:
# automatically gets a lock if we're in write/append mode
host, port = utils.get_host_port(_config['lockserver'])
self.lock_id = int(utils.get_lock(filepath, host, port))
if 'c' in mode:
File._cache[filepath] = self
def __init__(self, filepath, mode='rtc'):
"""filepath: the path of the distant file
mode: take the same argument as mode argument of the global
open() + optional flag c (which mean store in cache).
"""
self.mode = mode
self.filepath = filepath
host, port = utils.get_host_port(_config['nameserver'])
self.srv = utils.get_server(filepath, host, port)
if self.srv is None:
raise DFSIOError('Impossible to find a server that serve %s.'
% filepath)
self.last_modified = None
SpooledTemporaryFile.__init__(self, _config['max_size'], mode.replace('c', ''))
host, port = utils.get_host_port(_config['lockserver'])
if utils.is_locked(filepath, host, port):
raise DFSIOError('The file %s is locked.' % filepath)
if 'w' not in mode:
host, port = utils.get_host_port(self.srv)
with closing(HTTPConnection(host, port)) as con:
con.request('GET', filepath)
response = con.getresponse()
self.last_modified = response.getheader('Last-Modified')
status = response.status
if status not in (200, 204):
raise DFSIOError('Error (%d) while opening file.' % status)
if status != 204:
self.write(response.read())
if 'r' in mode:
self.seek(0)
self.lock_id = None
if 'a' in mode or 'w' in mode:
# automatically gets a lock if we're in write/append mode
host, port = utils.get_host_port(_config['lockserver'])
self.lock_id = int(utils.get_lock(filepath, host, port))
if 'c' in mode:
File._cache[filepath] = self
def __init__(self, filepath, mode='rtc'):
self.mode = mode
self.filepath = filepath
host, port = utils.get_host_port(_config['nameserver'])
self.srv = utils.get_server(filepath, host, port)
self.last_modified = None
SpooledTemporaryFile.__init__(self, _config['max_size'], mode.replace('c',''))
host, port = utils.get_host_port(_config['lockserver'])
if utils.is_locked(filepath, host, port):
raise DFSIOError('The file %s is locked.' % filepath)
if 'w' not in mode:
host, port = utils.get_host_port(self.srv)
with closing(HTTPConnection(host, port)) as con:
con.request('GET', filepath)
response = con.getresponse()
self.last_modified = response.getheader('Last-Modified')
status = response.status
if status not in (200, 204):
raise DFSIOError('Error (%d)while opening file.' % status)
if status != 204:
self.write(response.read())
if 'r' in mode:
self.seek(0)
self.lock_id = None
if 'a' in mode or 'w' in mode: # automatically lock file if appending or writing in file
host, port = utils.get_host_port(_config['lockserver'])
self.lock_id = int(utils.get_lock(filepath, host, port))
if 'c' in mode:
File._cache[filepath] = self
def __init__(self, path, mode='rtc'):
self.mode = mode
self.path = path
host, port = utils.get_host_port(_config['nameserver'])
self.srv = utils.get_server(path, host, port)
if self.srv is None:
pass
self.last_modified = None
SpooledTemporaryFile.__init__(self, _config['max_size'],
mode.replace('c', ''))
host, port = utils.get_host_port(_config['lockserver'])
if utils.is_locked(path, host, port):
pass
if 'w' not in mode:
host, port = utils.get_host_port(self.srv)
with closing(HTTPConnection(host, port)) as con:
con.request('GET', path)
response = con.getresponse()
self.last_modified = response.getheader('Last-Modified')
status = response.status
if status != 204:
self.write(response.read())
if 'r' in mode:
self.seek(0)
self.lock_id = None
if 'a' in mode or 'w' in mode:
host, port = utils.get_host_port(_config['lockserver'])
self.lock_id = int(utils.get_lock(path, host, port))
if 'c' in mode:
File._cache[path] = self
def upload(request):
if is_locked(request):
return HttpResponseRedirect(reverse('index'))
file = request.FILES['file'].readlines()
file_csv = csv.reader(file)
urls_to_add = []
for row in file_csv:
url = row[0]
try:
validate(url)
except Exception:
continue
# If we are here, we have a valid url
try:
article_type = row[3].strip()
except Exception:
# Could be an index error, could not None
article_type = ''
urls_to_add.append({
'url': url,
'archive': True if article_type == 'Archive' else False
})
add_urls.delay(
urls_to_add,
request.session['oauth_token'],
request.session['oauth_secret'])
lock(request.session['oauth_token'])
request.session['len_urls'] = len(urls_to_add)
return HttpResponseRedirect(reverse('done'))