def test_duplicatefilemd5_diff_resourcequeue(self): """ test_duplicatefilemd5_diff_resourcequeue 测试入库相同filemd5软件版本(crawl_package_version) 但下载队列记录(crawl_resource_queue)不同的软件版本(文件MD5唯一键冲突), 抛出异常中断,更新下载队列记录为相同文件md5 """ qitem = self.test_normal_storage() apsp= self._apkputstoragepipcls() try: apsp.process_item( item=qitem ) except DropItem, e: rrp = RecoverResourcePipeline() _item = rrp.process_item( item=qitem ) self.assertEquals( 5001 , _item['res_state'] )
def test_duplicatefilemd5_diff_resourcequeue(self): """ test_duplicatefilemd5_diff_resourcequeue 测试入库相同filemd5软件版本(crawl_package_version) 但下载队列记录(crawl_resource_queue)不同的软件版本(文件MD5唯一键冲突), 抛出异常中断,更新下载队列记录为相同文件md5 """ qitem = self.test_normal_storage() apsp = self._apkputstoragepipcls() try: apsp.process_item(item=qitem) except DropItem, e: rrp = RecoverResourcePipeline() _item = rrp.process_item(item=qitem) self.assertEquals(5001, _item['res_state'])
def test_exception_fileparse(self): """ test_exception_fileparse 解析文件数据错误-文件不存在,抛出异常中断操作 """ self.fixture_idx = 4 item = self.test_normal_download() src = os.path.join( settings.DOWNLOAD_TMP_DIR , item['identifier'] + ".apk" ) dst = os.path.join( settings.DOWNLOAD_TMP_DIR , item['identifier'] + ".apk.bk") os.rename( src , dst ) pipeline = self._fileparsepipcls() self.assertRaises( DropItem , pipeline.process_item , item ) os.rename( dst, src ) rrp = RecoverResourcePipeline() chitem = rrp.process_item( item ) self.assertEquals( 3001 , chitem['res_state'] )
def test_exception_download(self): """ test_exception_download 异常下载,抛出错误并中断,更新下载队列状态""" pipeline = self._downloadpipcls() item = self.test_normal_signin() item['download_url'] = 'http://apk.hiapk.com/404' self.assertRaises(DropItem, pipeline.process_item, item) fp = os.path.join(settings.DOWNLOAD_TMP_DIR, item['identifier'] + ".apk") self.assertFalse(os.path.isfile(fp)) self._rm_downloaditem = item rrp = RecoverResourcePipeline() chitem = rrp.process_item(item) self.assertEquals(2001, chitem['res_state']) return item
def test_exception_download(self): """ test_exception_download 异常下载,抛出错误并中断,更新下载队列状态""" pipeline = self._downloadpipcls() item = self.test_normal_signin() item['download_url'] = 'http://apk.hiapk.com/404'; self.assertRaises( DropItem , pipeline.process_item , item ) fp = os.path.join( settings.DOWNLOAD_TMP_DIR, item['identifier'] + ".apk" ) self.assertFalse( os.path.isfile(fp) ) self._rm_downloaditem = item rrp = RecoverResourcePipeline() chitem = rrp.process_item( item ) self.assertEquals( 2001 , chitem['res_state'] ) return item
def test_exception_fileparse(self): """ test_exception_fileparse 解析文件数据错误-文件不存在,抛出异常中断操作 """ self.fixture_idx = 4 item = self.test_normal_download() src = os.path.join(settings.DOWNLOAD_TMP_DIR, item['identifier'] + ".apk") dst = os.path.join(settings.DOWNLOAD_TMP_DIR, item['identifier'] + ".apk.bk") os.rename(src, dst) pipeline = self._fileparsepipcls() self.assertRaises(DropItem, pipeline.process_item, item) os.rename(dst, src) rrp = RecoverResourcePipeline() chitem = rrp.process_item(item) self.assertEquals(3001, chitem['res_state'])
def test_normal_signout(self): """ test_normal_signout 下载队列正常签出 """ pipeline = self._signoutpipcls() item = self.test_normal_fileparse() item = pipeline.process_item( item ) rrp = RecoverResourcePipeline() chitem = rrp.process_item( item ) self.assertEquals( 1001 , chitem['res_state'] ) self.assertEquals( settings.DOWNLOAD_TMP_DIR , chitem['fileinfo']['file_path'] ) self.assertEquals( chitem['fileinfo']['file_name'] , chitem['identifier'] + ".apk") self.assertEquals( 32 , len(chitem['fileinfo']['file_md5'])) self.assertGreater( item['fileinfo']['file_size'] , 10 ) return item
def test_normal_signout(self): """ test_normal_signout 下载队列正常签出 """ pipeline = self._signoutpipcls() item = self.test_normal_fileparse() item = pipeline.process_item(item) rrp = RecoverResourcePipeline() chitem = rrp.process_item(item) self.assertEquals(1001, chitem['res_state']) self.assertEquals(settings.DOWNLOAD_TMP_DIR, chitem['fileinfo']['file_path']) self.assertEquals(chitem['fileinfo']['file_name'], chitem['identifier'] + ".apk") self.assertEquals(32, len(chitem['fileinfo']['file_md5'])) self.assertGreater(item['fileinfo']['file_size'], 10) return item
def test_normal_archive(self): """ test_normal_archive 正常移动文件""" qitem = self.test_normal_storage() org_item = copy.deepcopy(qitem) pipeline = self._apkfilearchpipcls() qitem = pipeline.process_item( item=qitem ) self._rollback_resitem = ( org_item , qitem ) cpv = self.recover_pkgversion( qitem ) self.assertEquals( cpv['crawl_state'] , 1 ) rrp = RecoverResourcePipeline() _item = rrp.process_item( item=qitem ) self.assertEquals( _item['res_state'] , 1) self.assertEquals( cpv['file_name'] , _item['fileinfo']['file_name'] ) self.assertEquals( _item['fileinfo']['file_path'] , os.path.join( settings.APK_RESOURCE_REL_DIR , str(int(math.floor(cpv['res_id']/ pipeline.partition_limit))) ) )
def test_normal_archive(self): """ test_normal_archive 正常移动文件""" qitem = self.test_normal_storage() org_item = copy.deepcopy(qitem) pipeline = self._apkfilearchpipcls() qitem = pipeline.process_item(item=qitem) self._rollback_resitem = (org_item, qitem) cpv = self.recover_pkgversion(qitem) self.assertEquals(cpv['crawl_state'], 1) rrp = RecoverResourcePipeline() _item = rrp.process_item(item=qitem) self.assertEquals(_item['res_state'], 1) self.assertEquals(cpv['file_name'], _item['fileinfo']['file_name']) self.assertEquals( _item['fileinfo']['file_path'], os.path.join( settings.APK_RESOURCE_REL_DIR, str(int(math.floor(cpv['res_id'] / pipeline.partition_limit)))))