def on_download_file_completed(self, task):
"""
Called when the target file is fully downloaded.
This method retrieve the stored data and loop while all the metadata
have not been retrieved. That means that a file's matadata can be stored
in many chunks, the same way files are stored in vodstok.
"""
if self.__state == DownTask.SUMMARY:
filename, self.__chunks = self.__file.read().split('|')
if filename == 'metadata':
self.__file = MemoryStream('', key=self.__key)
self.__task = DownloadFileTask(self, self.__chunks.split(','), self.__file)
else:
self.__state = DownTask.RECVING
self.filename = clean_filename(os.path.join(self.__dst_prefix, filename))
self.__file = FileStream(
open(self.filename, 'wb'), key=self.__key
)
self.__task = DownloadFileTask(
self, self.__chunks.split(','), self.__file
)
if self.__manager is not None:
self.__manager.queue_task(self.__task)
elif self.__state == DownTask.RECVING:
self.__state = DownTask.DONE
self.__file.close()
if self.__manager is not None:
# notify manager of new servers
for chunk in self.__chunks.split(','):
server,alias = chunk.split('?')
self.__manager.on_server_discovered(Server(server))
self.__manager.on_task_done(self)
def __init__(self, manager=None, filename='unk.bin'):
try:
self.uuid = uuid.uuid1()
self.__manager = manager
self.__filename = clean_filename(filename)
self.__file = FileStream(open(filename, 'rb'))
self.__key = self.__file.get_key()
self.__state = UpTask.INIT
self.__task = UploadFileTask(self, self.__file)
self.__alias = None
except IOError:
raise IncorrectParameterError
def on_download_file_completed(self, task):
"""
Called when the target file is fully downloaded.
This method retrieve the stored data and loop while all the metadata
have not been retrieved. That means that a file's matadata can be stored
in many chunks, the same way files are stored in vodstok.
"""
if self.__state == DownTask.SUMMARY:
file_content = self.__file.read()
# Is it an old version of chunk ?
if (file_content.count('|') == 1):
filename, self.__chunks = file_content.split('|')
elif (file_content.count('|') == 2):
filename, version, self.__chunks = self.__file.read().split(
'|')
# Check version
# If version is greater than our version, raise an error.
if VersionStr(version) > Settings.version:
raise VersionError()
if filename == 'metadata':
self.__file = MemoryStream('', key=self.__key)
self.__task = DownloadFileTask(self,
self.__chunks.split(','),
self.__file)
else:
self.__state = DownTask.RECVING
self.filename = clean_filename(
os.path.join(self.__dst_prefix, filename))
self.__file = FileStream(open(self.filename, 'wb'),
key=self.__key)
self.__task = DownloadFileTask(self,
self.__chunks.split(','),
self.__file)
if self.__manager is not None:
self.__manager.queue_task(self.__task)
elif self.__state == DownTask.RECVING:
self.__state = DownTask.DONE
self.__file.close()
if self.__manager is not None:
# notify manager of new servers
for chunk in self.__chunks.split(','):
server, alias = chunk.split('?')
self.__manager.on_server_discovered(Server(server))
self.__manager.on_task_done(self)
def on_download_file_completed(self, task):
"""
Called when the target file is fully downloaded.
This method retrieve the stored data and loop while all the metadata
have not been retrieved. That means that a file's matadata can be stored
in many chunks, the same way files are stored in vodstok.
"""
if self.__state == DownTask.SUMMARY:
file_content = self.__file.read()
# Is it an old version of chunk ?
if (file_content.count('|') == 1):
filename, self.__chunks = file_content.split('|')
elif (file_content.count('|') == 2):
filename, version, self.__chunks = self.__file.read().split('|')
# Check version
# If version is greater than our version, raise an error.
if VersionStr(version) > Settings.version:
raise VersionError()
if filename == 'metadata':
self.__file = MemoryStream('', key=self.__key)
self.__task = DownloadFileTask(self, self.__chunks.split(','), self.__file)
else:
self.__state = DownTask.RECVING
self.filename = clean_filename(os.path.join(self.__dst_prefix, filename))
self.__file = FileStream(
open(self.filename, 'wb'), key=self.__key
)
self.__task = DownloadFileTask(
self, self.__chunks.split(','), self.__file
)
if self.__manager is not None:
self.__manager.queue_task(self.__task)
elif self.__state == DownTask.RECVING:
self.__state = DownTask.DONE
self.__file.close()
if self.__manager is not None:
# notify manager of new servers
for chunk in self.__chunks.split(','):
server, alias = chunk.split('?')
self.__manager.on_server_discovered(Server(server))
self.__manager.on_task_done(self)
def __init__(self, manager=None, filename='unk.bin'):
try:
self.uuid = uuid.uuid1()
self.__manager = manager
self.__filename = clean_filename(filename)
self.__file = FileStream(open(filename, 'rb'))
self.__key = self.__file.get_key()
self.__state = UpTask.INIT
self.__task = UploadFileTask(self, self.__file)
self.__alias = None
except IOError:
raise IncorrectParameterError
def __parse(self):
"""
Parse Vodstok download's URL and extract useful info
"""
r = urlparse.urlparse(self.__url)
self.__scheme = r.scheme
self.__server = r.netloc
#self.__key = self.__key.decode('hex')
self.__path = r.path
try:
self.__key, self.__chunk_id = r.fragment.split('-')
self.__key = self.__key.decode('hex')
self.__file = MemoryStream('', key=self.__key)
self.__alias = '%s://%s%s' % (self.__scheme, self.__server,
self.__path)
except ValueError:
raise IncorrectFormatError
def __parse(self):
"""
Parse Vodstok download's URL and extract useful info
"""
r = urlparse.urlparse(self.__url)
self.__scheme = r.scheme
self.__server = r.netloc
#self.__key = self.__key.decode('hex')
self.__path = r.path
try:
self.__key, self.__chunk_id = r.fragment.split('-')
self.__key = self.__key.decode('hex')
self.__file = MemoryStream('', key=self.__key)
self.__alias = '%s://%s%s' % (
self.__scheme,
self.__server,
self.__path
)
except ValueError:
raise IncorrectFormatError
class UpTask:
"""
Top-level upload handling class.
"""
INIT = -1
SENDING = 0
SUMMARY = 1
DONE = 2
def __init__(self, manager=None, filename='unk.bin'):
try:
self.uuid = uuid.uuid1()
self.__manager = manager
self.__filename = clean_filename(filename)
self.__file = FileStream(open(filename, 'rb'))
self.__key = self.__file.get_key()
self.__state = UpTask.INIT
self.__task = UploadFileTask(self, self.__file)
self.__alias = None
except IOError:
raise IncorrectParameterError
def set_manager(self, manager=None):
"""
Set task's manager
"""
self.__manager = manager
def process(self):
"""
Process upload: enqueue the corresponding task into our scheduler
"""
self.__state = UpTask.SENDING
if self.__manager is not None:
self.__manager.queue_task(self.__task)
def cancel(self):
"""
Cancel this task
"""
self.__task.cancel()
def suspend(self):
"""
Suspend this task
"""
self.__task.suspend()
def resume(self):
"""
Resume this task
"""
self.__task.resume()
def on_upload_file_completed(self, task):
"""
Called when all chunks have been uploaded.
Once all chunks were uploaded, we upload a metadata file (created in
memory) containing every chunks aliases and servers info. This metadata
may be store in multiple chunks, and this may be done recursively, while
the whole metadata cannot fit into a single chunk.
"""
if self.__state == UpTask.SENDING:
meta = '%s|%s' % (self.__filename, ','.join(task.get_aliases()))
if len(meta) > Settings.chunk_size:
meta = '%s|%s' % (self.__filename, ','.join(task.get_aliases()))
self.__filename = 'metadata'
if self.__manager is not None:
self.__task = UploadFileTask(
self, MemoryStream(meta, key=self.__key)
)
self.__manager.queue_task(self.__task)
else:
self.__state = UpTask.SUMMARY
self.__task = UploadFileTask(
self,
MemoryStream(meta, key=self.__key)
)
if self.__manager is not None:
self.__manager.queue_task(self.__task)
elif self.__state == UpTask.SUMMARY:
self.__state = UpTask.DONE
self.__alias = task.get_aliases()[0]
if self.__manager is not None:
self.__manager.on_task_done(self)
def get_url(self):
"""
Return the uploaded file's URL (link to the single chunk containing
the file's metadata.
"""
if self.__state == UpTask.DONE:
# split the alias
p = urlparse.urlparse(self.__alias)
k = self.__key.encode('hex')
return '%s://%s%s#%s-%s' % (p.scheme, p.netloc, p.path, k, p.query)
else:
return None
def on_progress(self, task, done, total):
"""
Notify upload progress
"""
if self.__state == UpTask.SENDING:
if self.__manager is not None:
self.__manager.on_task_progress(self, done, total)
def on_error(self, task):
"""
Called when the upload process failed
"""
self.__manager.on_task_error(self)
def get_key(self):
"""
Retrieve the upload encryption key
"""
return self.__key
class DownTask:
"""
This class wraps the download process and manages a download task. This
may be redundant with previous classes but this one is the top-level class
for download management. Only this one should be used when dealing with
download tasks.
"""
INIT = -1
SUMMARY = 0
RECVING = 1
DONE = 2
def __init__(self, manager=None, url='', dest_prefix=''):
self.uuid = uuid.uuid1()
self.__manager = manager
self.filename = None
self.__key = None
self.__alias = None
self.__url = url
self.__scheme = None
self.__chunk_id = None
self.__state = DownTask.INIT
self.__task = None
self.__dst_prefix = dest_prefix
self.__parse()
self.__chunks = None
def __parse(self):
"""
Parse Vodstok download's URL and extract useful info
"""
r = urlparse.urlparse(self.__url)
self.__scheme = r.scheme
self.__server = r.netloc
#self.__key = self.__key.decode('hex')
self.__path = r.path
try:
self.__key, self.__chunk_id = r.fragment.split('-')
self.__key = self.__key.decode('hex')
self.__file = MemoryStream('', key=self.__key)
self.__alias = '%s://%s%s' % (
self.__scheme,
self.__server,
self.__path
)
except ValueError:
raise IncorrectFormatError
def cancel(self):
"""
Cancel this task
"""
self.__task.cancel()
def suspend(self):
"""
Suspend this task.
"""
self.__task.suspend()
def resume(self):
"""
Resume this task
"""
self.__task.resume()
def set_manager(self, manager=None):
"""
Set this task's manager.
"""
self.__manager = manager
def process(self):
"""
Process download.
This method creates the download task and queue it into the scheduler.
"""
self.__state = DownTask.SUMMARY
#print self.__alias+'#'+self.__chunk_id
self.__task = DownloadFileTask(
self,
[
self.__alias+'?'+self.__chunk_id
],
self.__file
)
if self.__manager is not None:
self.__manager.queue_task(self.__task)
def on_download_file_completed(self, task):
"""
Called when the target file is fully downloaded.
This method retrieve the stored data and loop while all the metadata
have not been retrieved. That means that a file's matadata can be stored
in many chunks, the same way files are stored in vodstok.
"""
if self.__state == DownTask.SUMMARY:
filename, self.__chunks = self.__file.read().split('|')
if filename == 'metadata':
self.__file = MemoryStream('', key=self.__key)
self.__task = DownloadFileTask(self, self.__chunks.split(','), self.__file)
else:
self.__state = DownTask.RECVING
self.filename = clean_filename(os.path.join(self.__dst_prefix, filename))
self.__file = FileStream(
open(self.filename, 'wb'), key=self.__key
)
self.__task = DownloadFileTask(
self, self.__chunks.split(','), self.__file
)
if self.__manager is not None:
self.__manager.queue_task(self.__task)
elif self.__state == DownTask.RECVING:
self.__state = DownTask.DONE
self.__file.close()
if self.__manager is not None:
# notify manager of new servers
for chunk in self.__chunks.split(','):
server,alias = chunk.split('?')
self.__manager.on_server_discovered(Server(server))
self.__manager.on_task_done(self)
def on_progress(self, task, done, total):
"""
Notify download progress
"""
if self.__state == DownTask.RECVING:
if self.__manager is not None:
self.__manager.on_task_progress(self, done, total)
def on_error(self, task):
"""
Called when an error occurs while downloading
"""
self.__manager.on_task_error(self)
class UpTask:
"""
Top-level upload handling class.
"""
INIT = -1
SENDING = 0
SUMMARY = 1
DONE = 2
def __init__(self, manager=None, filename='unk.bin'):
try:
self.uuid = uuid.uuid1()
self.__manager = manager
self.__filename = clean_filename(filename)
self.__file = FileStream(open(filename, 'rb'))
self.__key = self.__file.get_key()
self.__state = UpTask.INIT
self.__task = UploadFileTask(self, self.__file)
self.__alias = None
except IOError:
raise IncorrectParameterError
def set_manager(self, manager=None):
"""
Set task's manager
"""
self.__manager = manager
def process(self):
"""
Process upload: enqueue the corresponding task into our scheduler
"""
self.__state = UpTask.SENDING
if self.__manager is not None:
self.__manager.queue_task(self.__task)
def cancel(self):
"""
Cancel this task
"""
self.__task.cancel()
def suspend(self):
"""
Suspend this task
"""
self.__task.suspend()
def resume(self):
"""
Resume this task
"""
self.__task.resume()
def on_upload_file_completed(self, task):
"""
Called when all chunks have been uploaded.
Once all chunks were uploaded, we upload a metadata file (created in
memory) containing every chunks aliases and servers info. This metadata
may be store in multiple chunks, and this may be done recursively, while
the whole metadata cannot fit into a single chunk.
"""
if self.__state == UpTask.SENDING:
meta = '%s|%s|%s' % (self.__filename, Settings.version, ','.join(
task.get_aliases()))
if len(meta) > Settings.chunk_size:
meta = '%s|%s|%s' % (self.__filename, Settings.version,
','.join(task.get_aliases()))
self.__filename = 'metadata'
if self.__manager is not None:
self.__task = UploadFileTask(
self, MemoryStream(meta, key=self.__key))
self.__manager.queue_task(self.__task)
else:
self.__state = UpTask.SUMMARY
self.__task = UploadFileTask(
self, MemoryStream(meta, key=self.__key))
if self.__manager is not None:
self.__manager.queue_task(self.__task)
elif self.__state == UpTask.SUMMARY:
self.__state = UpTask.DONE
self.__alias = task.get_aliases()[0]
if self.__manager is not None:
self.__manager.on_task_done(self)
def get_url(self):
"""
Return the uploaded file's URL (link to the single chunk containing
the file's metadata.
"""
if self.__state == UpTask.DONE:
# split the alias
p = urlparse.urlparse(self.__alias)
k = self.__key.encode('hex')
return '%s://%s%s#%s-%s' % (p.scheme, p.netloc, p.path, k, p.query)
else:
return None
def on_progress(self, task, done, total):
"""
Notify upload progress
"""
if self.__state == UpTask.SENDING:
if self.__manager is not None:
self.__manager.on_task_progress(self, done, total)
def on_error(self, task):
"""
Called when the upload process failed
"""
self.__manager.on_task_error(self)
def get_key(self):
"""
Retrieve the upload encryption key
"""
return self.__key
class DownTask:
"""
This class wraps the download process and manages a download task. This
may be redundant with previous classes but this one is the top-level class
for download management. Only this one should be used when dealing with
download tasks.
"""
INIT = -1
SUMMARY = 0
RECVING = 1
DONE = 2
def __init__(self, manager=None, url='', dest_prefix=''):
self.uuid = uuid.uuid1()
self.__manager = manager
self.filename = None
self.__key = None
self.__alias = None
self.__url = url
self.__scheme = None
self.__chunk_id = None
self.__state = DownTask.INIT
self.__task = None
self.__dst_prefix = dest_prefix
self.__parse()
self.__chunks = None
def __parse(self):
"""
Parse Vodstok download's URL and extract useful info
"""
r = urlparse.urlparse(self.__url)
self.__scheme = r.scheme
self.__server = r.netloc
#self.__key = self.__key.decode('hex')
self.__path = r.path
try:
self.__key, self.__chunk_id = r.fragment.split('-')
self.__key = self.__key.decode('hex')
self.__file = MemoryStream('', key=self.__key)
self.__alias = '%s://%s%s' % (self.__scheme, self.__server,
self.__path)
except ValueError:
raise IncorrectFormatError
def cancel(self):
"""
Cancel this task
"""
self.__task.cancel()
def suspend(self):
"""
Suspend this task.
"""
self.__task.suspend()
def resume(self):
"""
Resume this task
"""
self.__task.resume()
def set_manager(self, manager=None):
"""
Set this task's manager.
"""
self.__manager = manager
def process(self):
"""
Process download.
This method creates the download task and queue it into the scheduler.
"""
self.__state = DownTask.SUMMARY
#print self.__alias+'#'+self.__chunk_id
self.__task = DownloadFileTask(self,
[self.__alias + '?' + self.__chunk_id],
self.__file)
if self.__manager is not None:
self.__manager.queue_task(self.__task)
def on_download_file_completed(self, task):
"""
Called when the target file is fully downloaded.
This method retrieve the stored data and loop while all the metadata
have not been retrieved. That means that a file's matadata can be stored
in many chunks, the same way files are stored in vodstok.
"""
if self.__state == DownTask.SUMMARY:
file_content = self.__file.read()
# Is it an old version of chunk ?
if (file_content.count('|') == 1):
filename, self.__chunks = file_content.split('|')
elif (file_content.count('|') == 2):
filename, version, self.__chunks = self.__file.read().split(
'|')
# Check version
# If version is greater than our version, raise an error.
if VersionStr(version) > Settings.version:
raise VersionError()
if filename == 'metadata':
self.__file = MemoryStream('', key=self.__key)
self.__task = DownloadFileTask(self,
self.__chunks.split(','),
self.__file)
else:
self.__state = DownTask.RECVING
self.filename = clean_filename(
os.path.join(self.__dst_prefix, filename))
self.__file = FileStream(open(self.filename, 'wb'),
key=self.__key)
self.__task = DownloadFileTask(self,
self.__chunks.split(','),
self.__file)
if self.__manager is not None:
self.__manager.queue_task(self.__task)
elif self.__state == DownTask.RECVING:
self.__state = DownTask.DONE
self.__file.close()
if self.__manager is not None:
# notify manager of new servers
for chunk in self.__chunks.split(','):
server, alias = chunk.split('?')
self.__manager.on_server_discovered(Server(server))
self.__manager.on_task_done(self)
def on_progress(self, task, done, total):
"""
Notify download progress
"""
if self.__state == DownTask.RECVING:
if self.__manager is not None:
self.__manager.on_task_progress(self, done, total)
def on_error(self, task):
"""
Called when an error occurs while downloading
"""
self.__manager.on_task_error(self)
