def get_file(ftp_dir,storage_path,name, bundlename):
log_info('Fetching file [%s]' % name)
create_dir(user_file_cache)
ftp = None
try:
ftp = FTP(FREABES_FTP,timeout=10)
if not os.path.isfile(storage_path):
ftp.login(USERNAME,PASSWORD)
ftp.cwd(ftp_dir)
file_to_write = open(storage_path, 'wb')
size = ftp.size(name)
ftp.retrbinary('RETR %s' % name, lambda block: flush(block, file_to_write, ftp, storage_path, size, bundlename))
else:
log_info('Already fetched file %s' % name)
except ftplib.all_errors as a:
if is_ftp_overloaded(a):
raise FtpOverloadedException('overload')
else:
raise InvalidZipFileException('Unable to get file')
except:
if os.path.exists(storage_path):
log_info('Removing file [%s]' %storage_path)
delete_file(storage_path)
if not (was_dia_cancelled()):
log_info('Exception while retrieving file %s , retrying' %name)
raise InvalidZipFileException('Unable to get file')
else:
log_info('Cancelled download')
finally:
if ftp is not None:
ftp.close()
def flush(block, fileToWrite ,ftp , storage_path, size, name):
global log_ctr
if was_dia_cancelled():
fileToWrite.close()
raise Exception('User cancel')
else:
fileToWrite.write(block)
if log_ctr % 2000 == 0:
progress = (float(os.path.getsize(storage_path)) / size)*100
log_dia('Fetching zipfile for %s, progress %.1f%%' %(name, progress))
log_ctr = log_ctr + 1
def install_all(self):
sleep(1)
ticks = 0
for entry in db:
if not utils.has_attribute('exclude_%s' % entry.name):
if entry.needs_update and entry.create_delta is True or entry.not_installed or entry.create_delta is False:
ticks += 1
dia_set_ticks(ticks)
dia_tick()
for entry in db:
if not was_dia_cancelled():
if not utils.has_attribute('exclude_%s' % entry.name):
if entry.needs_update and entry.create_delta is True:
update_item(self,entry,False)
update_tree_view(self)
elif entry.not_installed or entry.create_delta is False:
install_item(self,entry,False)
update_tree_view(self)
wx.CallAfter(dispose_dialog)
def extract_file(location, target,curr_item,was_install):
fh = open(location, 'rb')
was_cancelled = False
ctr = 0
try:
z = zipfile.ZipFile(fh)
for name in z.namelist():
try:
outpath = target
z.extract(name, outpath)
if ctr % UPDATE_LOG_DIA_EVERY == 0:
log_dia('Extracting %s' % name)
if ctr % CHECK_REMAINING_STORAGE_EVERY == 0:
if get_free_space_mb(target) < MIN_DISK_MB:
wx.MessageBox('Not enough space left on device, cancelling', 'Info',wx.OK | wx.ICON_WARNING)
was_cancelled = True
break
if was_dia_cancelled():
was_cancelled = True
break
ctr += 1
except:
log_dia_info('unable to extract [%s]' %name)
except:
raise InvalidZipFileException('Invalid zipentry')
finally:
fh.close()
if was_cancelled and was_install:
folder = '%s/%s' % (target,curr_item.folder)
log_dia_info('Cleaned up folder %s after cancel' % folder)
delete_folder(folder)
curr_item.was_installed = False
delete_file(resolveDbFile(target,curr_item.name))
log_dia_info('Done')
def update_item(self, item,dispose_dia = True):
post_ga_event('update_item','artifact_%s' % item.name)
try:
if item.create_delta:
for deltaitem in item.deltas:
if not utils.check_if_already_updated_with_delta(dir_name, item.name, deltaitem.version):
log_dia_info('Updating file %s' % deltaitem.file)
retries = 0
nof_retries = 3
while retries < nof_retries:
utils.get_file(utils.resolve_delta_dir(item),utils.get_storage_location(deltaitem.file),deltaitem.file,item.name)
dia_tick()
if was_dia_cancelled():
log_dia_info('Cancelling...')
break
if item.dynamic_import:
utils.delete_folder(utils.user_file_cache_dyn)
utils.create_dir(utils.user_file_cache_dyn)
try:
if was_dia_cancelled():
log_dia_info('Cancelling...')
break
utils.extract_file(utils.get_storage_location(deltaitem.file), utils.user_file_cache_dyn,item,True)
if was_dia_cancelled():
log_dia_info('Cancelling...')
break
dynamic_import.move_in_place(utils.user_file_cache + 'dyn/%s/' % item.folder, '%s/%s/' % (dir_name, item.folder))
if was_dia_cancelled():
log_dia_info('Cancelling...')
update_tree_view(self)
break
utils.update_db(dir_name, item.name, deltaitem.version)
utils.delete_folder(utils.user_file_cache + 'dyn/%s/' % item.folder)
item.needs_update = False
update_tree_view(self)
dia_tick()
break
except FtpOverloadedException:
post_ga_event('update_item_ftp_overload','artifact_%s' % deltaitem.file)
raise
except InvalidZipFileException as e:
post_ga_event('update_item_invalid_zip','artifact_%s' % deltaitem.file)
utils.delete_file(utils.get_storage_location(deltaitem.file))
if retries == nof_retries-1:
raise
else:
log_info('Unpacking %s into %s' % (item.name, dir_name))
try:
if was_dia_cancelled():
log_dia_info('Cancelling...')
break
utils.extract_file(utils.get_storage_location(deltaitem.file), dir_name,item,False)
if was_dia_cancelled():
log_dia_info('Cancelling...')
update_tree_view(self)
break
utils.update_db(dir_name, item.name, deltaitem.version)
target_folder = dir_name + '/' + item.folder
log_dia_info('Updated %s with deltafile %s at location %s' % (item.name,deltaitem.file,target_folder))
item.needs_update = False
update_tree_view(self)
dia_tick()
if utils.get_boolean_user_setting(delete_files_after_install):
utils.delete_file(utils.get_storage_location(deltaitem.file))
break
except FtpOverloadedException:
post_ga_event('update_item_ftp_overload','artifact_%s' % deltaitem.file)
raise
except InvalidZipFileException:
post_ga_event('update_item_invalid_zip','artifact_%s' % deltaitem.file)
log_dia_info('Invalid deltazifile, delete and retry')
utils.delete_file(utils.get_storage_location(deltaitem.file))
if retries == nof_retries-1:
raise
retries += 1
if dispose_dia:
wx.CallAfter(dispose_dialog)
except FtpOverloadedException:
if dispose_dia:
wx.CallAfter(dispose_dialog_fail,'Too many users right now, please try again later')
except InvalidZipFileException as e:
if dispose_dia:
wx.CallAfter(dispose_dialog_fail,e.message)
except:
if dispose_dia:
wx.CallAfter(dispose_dialog_fail,'Unknown error %s:' % sys.exc_info()[0])
finally:
update_tree_view(self)
def install_item(self, current_item, dispose_dia = True):
post_ga_event('install_item','artifact_%s' % current_item.name)
folder = '%s/%s/' % (dir_name, current_item.folder)
if not utils.check_if_already_updated_with_delta(dir_name, current_item.name, current_item.version) or not os.path.exists(folder):
try:
log_dia_info('Getting full entry %s' % current_item.name)
retries = 0
nof_retries = 3
while retries < nof_retries:
try:
utils.get_file(current_item.basedir,utils.get_storage_location(current_item.file),current_item.file,current_item.name)
if os.path.exists(folder):
log_dia_info('Deleting current folder %s (this may take a while, please be patient)' % folder)
utils.delete_folder('%s/%s/' % (dir_name, current_item.folder))
if was_dia_cancelled():
log_dia_info('Downloaded for %s was cancelled' % current_item.name)
if dispose_dia:
wx.CallAfter(dispose_dialog)
return
log_dia_info('Downloaded %s' % current_item.name)
log_dia_info('Extracting files %s' %dir_name)
dia_tick()
utils.extract_file(utils.get_storage_location(current_item.file), dir_name,current_item,True)
dia_tick()
break;
except InvalidZipFileException:
post_ga_event('install_item_invalid_zip','artifact_%s' % current_item.name)
log_info('Invalid zifile, delete and retry')
utils.delete_file(utils.get_storage_location(current_item.file))
if retries == nof_retries-1:
raise
retries+=1
if utils.get_boolean_user_setting(delete_files_after_install):
utils.delete_file(utils.get_storage_location(current_item.file))
if was_dia_cancelled():
if dispose_dia:
wx.CallAfter(dispose_dialog)
return
log_dia_info('Update db')
utils.update_db(dir_name, current_item.name, current_item.version)
current_item.not_installed = False
log_dia_info('Done extracting full entry %s at location %s' % (current_item.name, dir_name))
log_dia_info('Install done')
if dispose_dia:
wx.CallAfter(dispose_dialog)
except InvalidZipFileException as e:
utils.delete_folder('%s/%s/' % (dir_name, current_item.folder))
log_dia_info('Install failed du to error during fetch or unzip')
if dispose_dia:
wx.CallAfter(dispose_dialog_fail,e.message)
except FtpOverloadedException as e:
log_dia_info('Too many users, please try agin in a while')
if dispose_dia:
wx.CallAfter(dispose_dialog_fail,'Too many users, please try later')
except:
utils.delete_folder('%s/%s/' % (dir_name, current_item.folder))
log_dia_info('Install failed due to unknown error')
if dispose_dia:
wx.CallAfter(dispose_dialog_fail,'Unknown error %s:' % sys.exc_info()[0])
finally:
update_tree_view(self)
else:
if dispose_dia:
wx.CallAfter(dispose_dialog)
