def download(node_id: str, local_path: str, exclude: list) -> int: node = query.get_node(node_id) if not node.is_available(): return 0 if node.is_folder(): return download_folder(node_id, local_path, exclude) loc_name = node.name # # downloading a non-cached node # if not loc_name: # loc_name = node_id for reg in exclude: if re.match(reg, loc_name): print('Skipping download of "%s" because of exclusion pattern.' % loc_name) return 0 hasher = hashing.IncrementalHasher() try: print('Current file: %s' % loc_name) content.download_file(node_id, loc_name, local_path, length=node.size, write_callback=hasher.update) except RequestError as e: logger.error('Downloading "%s" failed. Code: %s, msg: %s' % (loc_name, e.status_code, e.msg)) return UL_DL_FAILED return compare_hashes(hasher.get_result(), node.md5, loc_name)
def test_download(self): fn, sz = gen_rand_file() self.assertTrue(sz < content.CONSECUTIVE_DL_LIMIT) md5 = hashing.hash_file(fn) n = content.upload_file(fn) self.assertIn('id', n) os.remove(fn) self.assertFalse(os.path.exists(fn)) content.download_file(n['id'], fn) md5_dl = hashing.hash_file(fn) self.assertEqual(md5, md5_dl) trash.move_to_trash(n['id']) os.remove(fn)
def test_incomplete_download(self): ch_sz = gen_rand_sz() content.CHUNK_SIZE = ch_sz fn, sz = gen_rand_file(size=5 * ch_sz) md5 = hashing.hash_file(fn) n = content.upload_file(fn) self.assertEqual(n['contentProperties']['md5'], md5) os.remove(fn) self.assertFalse(os.path.exists(fn)) with self.assertRaises(RequestError) as cm: content.download_file(n['id'], fn, length=sz + 1) #os.remove(fn + content.PARTIAL_SUFFIX) self.assertEqual(cm.exception.status_code, RequestError.CODE.INCOMPLETE_RESULT) content.download_file(n['id'], fn, length=sz) os.remove(fn)
def download_file(node_id: str, local_path: str, pg_handler: progress.FileProgress=None) -> RetryRetVal: node = query.get_node(node_id) name, md5, size = node.name, node.md5, node.size # db.Session.remove() # otherwise, sqlalchemy will complain if thread crashes logger.info('Downloading "%s"' % name) hasher = hashing.IncrementalHasher() try: content.download_file(node_id, name, local_path, length=size, write_callbacks=[hasher.update, pg_handler.update]) except RequestError as e: logger.error('Downloading "%s" failed. Code: %s, msg: %s' % (name, e.status_code, e.msg)) return UL_DL_FAILED else: return compare_hashes(hasher.get_result(), md5, name)
def test_incomplete_download(self): ch_sz = gen_rand_sz() content.CHUNK_SIZE = ch_sz fn, sz = gen_rand_file(size=5 * ch_sz) md5 = hashing.hash_file(fn) n = content.upload_file(fn) self.assertEqual(n['contentProperties']['md5'], md5) os.remove(fn) self.assertFalse(os.path.exists(fn)) with self.assertRaises(RequestError) as cm: content.download_file(n['id'], fn, length=sz + 1) #os.remove(fn + content.PARTIAL_SUFFIX) self.assertEqual(cm.exception.status_code, RequestError.CODE.INCOMPLETE_RESULT) content.download_file(n['id'], fn, length=sz) trash.move_to_trash(n['id']) os.remove(fn)
def test_download_resume(self): ch_sz = gen_rand_sz() content.CHUNK_SIZE = ch_sz content.CONSECUTIVE_DL_LIMIT = ch_sz fn, sz = gen_rand_file(size=5 * ch_sz) md5 = hashing.hash_file(fn) n = content.upload_file(fn) self.assertEqual(n['contentProperties']['md5'], md5) os.remove(fn) self.assertFalse(os.path.exists(fn)) p_fn = fn + content.PARTIAL_SUFFIX with open(p_fn, 'wb') as f: content.chunked_download(n['id'], f, length=int(sz * random.random())) self.assertLess(os.path.getsize(p_fn), sz) content.download_file(n['id'], fn) trash.move_to_trash(n['id']) dl_md5 = hashing.hash_file(fn) self.assertEqual(md5, dl_md5) os.remove(fn)