def test_duplicatefilemd5_diff_resourcequeue(self):
""" test_duplicatefilemd5_diff_resourcequeue
测试入库相同filemd5软件版本(crawl_package_version)
但下载队列记录(crawl_resource_queue)不同的软件版本(文件MD5唯一键冲突),
抛出异常中断,更新下载队列记录为相同文件md5
"""
qitem = self.test_normal_storage()
apsp= self._apkputstoragepipcls()
try:
apsp.process_item( item=qitem )
except DropItem, e:
rrp = RecoverResourcePipeline()
_item = rrp.process_item( item=qitem )
self.assertEquals( 5001 , _item['res_state'] )
def test_duplicatefilemd5_diff_resourcequeue(self):
""" test_duplicatefilemd5_diff_resourcequeue
测试入库相同filemd5软件版本(crawl_package_version)
但下载队列记录(crawl_resource_queue)不同的软件版本(文件MD5唯一键冲突),
抛出异常中断,更新下载队列记录为相同文件md5
"""
qitem = self.test_normal_storage()
apsp = self._apkputstoragepipcls()
try:
apsp.process_item(item=qitem)
except DropItem, e:
rrp = RecoverResourcePipeline()
_item = rrp.process_item(item=qitem)
self.assertEquals(5001, _item['res_state'])
def test_exception_fileparse(self):
""" test_exception_fileparse 解析文件数据错误-文件不存在，抛出异常中断操作 """
self.fixture_idx = 4
item = self.test_normal_download()
src = os.path.join( settings.DOWNLOAD_TMP_DIR , item['identifier'] + ".apk" )
dst = os.path.join( settings.DOWNLOAD_TMP_DIR , item['identifier'] + ".apk.bk")
os.rename( src , dst )
pipeline = self._fileparsepipcls()
self.assertRaises( DropItem , pipeline.process_item , item )
os.rename( dst, src )
rrp = RecoverResourcePipeline()
chitem = rrp.process_item( item )
self.assertEquals( 3001 , chitem['res_state'] )
def test_exception_download(self):
""" test_exception_download 异常下载，抛出错误并中断，更新下载队列状态"""
pipeline = self._downloadpipcls()
item = self.test_normal_signin()
item['download_url'] = 'http://apk.hiapk.com/404'
self.assertRaises(DropItem, pipeline.process_item, item)
fp = os.path.join(settings.DOWNLOAD_TMP_DIR,
item['identifier'] + ".apk")
self.assertFalse(os.path.isfile(fp))
self._rm_downloaditem = item
rrp = RecoverResourcePipeline()
chitem = rrp.process_item(item)
self.assertEquals(2001, chitem['res_state'])
return item
def test_exception_download(self):
""" test_exception_download 异常下载，抛出错误并中断，更新下载队列状态"""
pipeline = self._downloadpipcls()
item = self.test_normal_signin()
item['download_url'] = 'http://apk.hiapk.com/404';
self.assertRaises( DropItem , pipeline.process_item , item )
fp = os.path.join( settings.DOWNLOAD_TMP_DIR,
item['identifier'] + ".apk" )
self.assertFalse( os.path.isfile(fp) )
self._rm_downloaditem = item
rrp = RecoverResourcePipeline()
chitem = rrp.process_item( item )
self.assertEquals( 2001 , chitem['res_state'] )
return item
def test_exception_fileparse(self):
""" test_exception_fileparse 解析文件数据错误-文件不存在，抛出异常中断操作 """
self.fixture_idx = 4
item = self.test_normal_download()
src = os.path.join(settings.DOWNLOAD_TMP_DIR,
item['identifier'] + ".apk")
dst = os.path.join(settings.DOWNLOAD_TMP_DIR,
item['identifier'] + ".apk.bk")
os.rename(src, dst)
pipeline = self._fileparsepipcls()
self.assertRaises(DropItem, pipeline.process_item, item)
os.rename(dst, src)
rrp = RecoverResourcePipeline()
chitem = rrp.process_item(item)
self.assertEquals(3001, chitem['res_state'])
def test_normal_signout(self):
""" test_normal_signout 下载队列正常签出 """
pipeline = self._signoutpipcls()
item = self.test_normal_fileparse()
item = pipeline.process_item( item )
rrp = RecoverResourcePipeline()
chitem = rrp.process_item( item )
self.assertEquals( 1001 , chitem['res_state'] )
self.assertEquals( settings.DOWNLOAD_TMP_DIR , chitem['fileinfo']['file_path'] )
self.assertEquals( chitem['fileinfo']['file_name'] ,
chitem['identifier'] + ".apk")
self.assertEquals( 32 , len(chitem['fileinfo']['file_md5']))
self.assertGreater( item['fileinfo']['file_size'] , 10 )
return item
def test_normal_signout(self):
""" test_normal_signout 下载队列正常签出 """
pipeline = self._signoutpipcls()
item = self.test_normal_fileparse()
item = pipeline.process_item(item)
rrp = RecoverResourcePipeline()
chitem = rrp.process_item(item)
self.assertEquals(1001, chitem['res_state'])
self.assertEquals(settings.DOWNLOAD_TMP_DIR,
chitem['fileinfo']['file_path'])
self.assertEquals(chitem['fileinfo']['file_name'],
chitem['identifier'] + ".apk")
self.assertEquals(32, len(chitem['fileinfo']['file_md5']))
self.assertGreater(item['fileinfo']['file_size'], 10)
return item
def test_normal_archive(self):
""" test_normal_archive 正常移动文件"""
qitem = self.test_normal_storage()
org_item = copy.deepcopy(qitem)
pipeline = self._apkfilearchpipcls()
qitem = pipeline.process_item( item=qitem )
self._rollback_resitem = ( org_item , qitem )
cpv = self.recover_pkgversion( qitem )
self.assertEquals( cpv['crawl_state'] , 1 )
rrp = RecoverResourcePipeline()
_item = rrp.process_item( item=qitem )
self.assertEquals( _item['res_state'] , 1)
self.assertEquals( cpv['file_name'] , _item['fileinfo']['file_name'] )
self.assertEquals( _item['fileinfo']['file_path'] ,
os.path.join( settings.APK_RESOURCE_REL_DIR ,
str(int(math.floor(cpv['res_id']/ pipeline.partition_limit))) )
)
def test_normal_archive(self):
""" test_normal_archive 正常移动文件"""
qitem = self.test_normal_storage()
org_item = copy.deepcopy(qitem)
pipeline = self._apkfilearchpipcls()
qitem = pipeline.process_item(item=qitem)
self._rollback_resitem = (org_item, qitem)
cpv = self.recover_pkgversion(qitem)
self.assertEquals(cpv['crawl_state'], 1)
rrp = RecoverResourcePipeline()
_item = rrp.process_item(item=qitem)
self.assertEquals(_item['res_state'], 1)
self.assertEquals(cpv['file_name'], _item['fileinfo']['file_name'])
self.assertEquals(
_item['fileinfo']['file_path'],
os.path.join(
settings.APK_RESOURCE_REL_DIR,
str(int(math.floor(cpv['res_id'] /
pipeline.partition_limit)))))
