def get_conf(url, download_dir, useThreads):
data_path = download_dir + '/download_data'
json_path = download_dir + '/download.json'
hashes_path = download_dir + '/hashes.json'
hashes = None
resuming = True
threadQueue = None
if useThreads:
threadQueue = ThreadQueue()
if not file_exists(download_dir):
os.mkdir(download_dir)
conf_new = json_from_url(url)
conf = shellutils.read_json(json_path)
if conf is None:
logger.info('No local download.json file found. Downloading download.json from server.')
conf = conf_new
resuming = False
else:
logger.info('download.json read successfully')
if not file_exists(hashes_path):
logger.info('No local hashes.json file found.')
resuming = False
else:
logger.info('Local hashes file found.')
new_conf_ver = int(conf_new['version'])
conf_ver = int(conf['version'])
if new_conf_ver > conf_ver or new_conf_ver == -1:
conf = conf_new
logger.info('download.json on server is newer than local version. Restarting download process.')
resuming = False
else:
logger.info('Local download.json is the newest version.')
#have good conf now
raw_f_name = conf['raw-file']
raw_path = download_dir + '/' + raw_f_name
raw_url = '/'.join(url.split('/')[:-1]) + '/' + raw_f_name
#bro_url = raw_url + '.brotli'
logger.info('raw file url: %s' % raw_url)
#compress = int(conf['compression'])
#if compress != 0 and compress != 1:
# logger.critical('Unsupported compression type: %i.' % compress)
#if compress == 1:
# import brotli
# url_to_download = bro_url
return conf, raw_path, json_path, data_path, hashes_path, hashes, resuming, threadQueue, raw_url
def get_working_dir(self):
if not self.is_running():
return None
path = u.expand_link(u.join('/proc', self.pid, 'cwd'))
if u.file_exists(path):
return path
else:
return None
def get_working_dir(self):
if not self.is_running():
return None
path = u.expand_link(u.join('/proc', self.pid, 'cwd'))
if u.file_exists(path):
return path
else:
return None
def download(json_url, download_dir, onProgress, useThreads=False): #onComplete?
conf, raw_path, json_path, data_path, hashes_path, hashes, resuming, threadQueue, raw_url = get_conf(json_url, download_dir, useThreads)
write_block = write_block_
if threadQueue is not None:
write_block = lambda *args: threadQueue.add_task(write_block_, *args)
if not resuming:
shellutils.write_json(json_path, conf)
hashes = json_from_url(json_url + '.hashes')
shellutils.write_json(hashes_path, hashes)
if file_exists(raw_path):
shellutils.rm(raw_path)
else:
hashes = shellutils.read_json(hashes_path)
progress = conf['progress']
num_hashes = conf['num-hashes']
blk_size = conf['block-size']
final_file = open(raw_path, 'ab')
while progress < num_hashes:
if bad_block is not None:
msg = 'Bad hash of download chunk %s' % bad_block
logger.critical(msg)
raise Status(status.DOWNLOAD, msg)
offset = progress * blk_size
end = offset + blk_size - 1
header = { 'Range': 'bytes=%d-%d' % (offset, end) }
#Stream=False makes it faster
r = requests.get(raw_url, headers=header, stream=False, verify=True, allow_redirects=True)
if r.status_code != 206:
msg = 'Wrong code for getting chunk. Got %i' % r.status_code
raise Status(status.DOWNLOAD, msg)
data = r.content
r.close()
write_block(data, hashes, final_file, offset, conf, progress, json_path)
progress = progress + 1
onProgress(progress, num_hashes)
if threadQueue is not None:
logger.info('Waiting on writer threads')
threadQueue.join()
logger.info('Writer threads are done')
#final_file.flush()
final_file.close()
if bad_block is None:
logger.info('Download done!')
return conf, raw_path, json_path
else:
msg = 'Have bad block %s' % bad_block
logger.info(msg)
raise Status(status.DOWNLOAD, msg)
def get_exe_path(self):
if not self.is_running():
return None
try:
path = u.expand_link(u.join('/proc', self.pid, 'exe'))
except:
return None
if u.file_exists(path):
return path
else:
return None
def get_exe_path(self):
if not self.is_running():
return None
try:
path = u.expand_link(u.join('/proc', self.pid, 'exe'))
except:
return None
if u.file_exists(path):
return path
else:
return None
def get_conf(serv_url, download_dir):
if not file_exists(download_dir):
shellutils.mkdir(download_dir)
ret = get_latest_json(serv_url, download_dir, conf_fname)
conf, latest_conf = ret
ret = get_latest_json(serv_url, download_dir, hashes_fname)
hashes, latest_hashes = ret
resuming = False
if latest_conf and latest_hashes:
resuming = True
logger.info('Everything is up to date. Resuming.')
else:
logger.info('Files outdated. Restarting.')
raw_f_name = conf['raw-file']
raw_path = join(download_dir, raw_f_name)
raw_url = urljoin(serv_url, raw_f_name)
logger.info('raw file url: %s' % raw_url)
#todo: left here
return conf, raw_path, json_path, data_path, hashes_path, hashes, resuming, threadQueue, raw_url
inputs = {
#'DSC_5747.MOV' : 'Hack1_income_vs_assets.mp4',
#'DSC_5748.MOV' : 'Hack2_habits.mp4',
#'DSC_5751.MOV' : 'Hack3_revocable_living_trust.mp4',
#'DSC_5752.MOV' : 'Hack4_bigfoot.mp4',
'DSC_5754.MOV': 'Hack5_401K_help.mp4',
#'DSC_5755.MOV' : 'Hack6_cost_segregation.mp4',
#'DSC_5756.MOV' : 'Hack7_tax_14_day.mp4',
'DSC_5758.MOV': 'Hack8_kiddie_tax.mp4',
#'DSC_5759.MOV' : 'Hack9_medicaid_nursing_home.mp4'
}
bad = False
for in_name in inputs:
if not shellutils.file_exists(in_name):
bad = True
print(in_name)
if bad:
sys.exit()
def print_out(line):
#print(line)
pass
convert_cmd = sh.Command('./convert.sh')
for in_name in inputs:
def download(url, download_dir, onProgress, useThreads=False):
conf, raw_path, json_path, data_path, hashes_path, hashes, resuming, threadQueue, url_to_download = get_conf(
url, download_dir, useThreads)
if not resuming:
if file_exists(data_path):
shutil.rmtree(data_path)
os.mkdir(data_path)
write_conf(json_path, conf)
hashes = json_from_url(url + '.hashes')
write_conf(hashes_path, hashes)
else:
hashes = read_conf(hashes_path)
chunk_status, progress_tracker = check_data(data_path, hashes)
block_size = conf['block-size']
num_hashes = conf['num-hashes']
download_done = False
while not download_done:
for x in chunk_status:
if x[1] is True:
continue
start = block_size * get_chunk_index(chunk_status, x[0])
end = start + block_size - 1
resume_header = {'Range': 'bytes=%d-%d' % (start, end)}
r = requests.get(url_to_download,
headers=resume_header,
stream=True,
verify=True,
allow_redirects=True)
if r.status_code != 206:
err = 'Could not find update file on server'
logger.critical(err)
raise Exception(err)
data = b''
for c in r.iter_content(block_size):
data += c
#if compress == 1:
# data = brotli.decompress(data)
chunk_hash = check_and_write_chunk(hashes, x[0], data, data_path,
threadQueue)
if chunk_hash is not None:
set_chunk_status(chunk_status, chunk_hash, True)
onProgress(progress_tracker, num_hashes)
progress_tracker = progress_tracker + 1
r.close()
download_done = True
for x in chunk_status: #chunk_status.items()
if x[1] is False:
download_done = False
logger.info('Download done! Now putting stuff together')
if threadQueue is not None:
threadQueue.join()
with open(raw_path, 'wb') as final_file:
for x in hashes:
with open(data_path + '/' + str(x), 'rb') as chunk:
final_file.write(chunk.read())
return conf, raw_path, json_path
