def test_update_analysis(self): # create not existent analysis_id = '016b792f-e659-4143-b833-163141e21363' self.assertFalse(Analysis.objects.filter( analysis_id=analysis_id).exists()) api_request = RequestMinimal(query={'analysis_id': analysis_id}) result = api_request.call().next() analysis = update_analysis(result) self.assertTrue(Analysis.objects.filter( analysis_id=analysis_id).exists()) # update new_last_modified = '2077-77-77T77:77:77Z' new_center_name = 'ABCD' self.assertNotEqual(analysis.last_modified, new_last_modified) self.assertNotEqual(analysis.center_name, new_center_name) analysis.last_modified = new_last_modified analysis.center_name = new_center_name analysis.save() self.assertEqual(analysis.last_modified, new_last_modified) self.assertEqual(analysis.center_name, new_center_name) analysis = update_analysis(result) self.assertNotEqual(analysis.last_modified, new_last_modified) self.assertNotEqual(analysis.center_name, new_center_name) # check all attributes are setted api_request = RequestDetailJSON(query={'analysis_id': analysis_id}) result = api_request.call().next() for attr in CART_SORT_ATTRIBUTES: self.assertEqual(result.get(attr), getattr(analysis, attr))
def handle(self, *args, **options): self.stdout.write('Searching for outdated analysises ...\n') self.stderr.write('Searching for outdated analysises ...\n') api_request = RequestMinimal(query={}) for result in api_request.call(): try: analysis = Analysis.objects.get( analysis_id=result['analysis_id'], last_modified=result['last_modified']) if analysis.last_modified != result['last_modified']: analysis = update_analysis(result) if analysis: self.stderr.write('- %s was updated\n' % result['analysis_id']) else: self.stderr.write('- %s was skipped\n' % result['analysis_id']) except Analysis.DoesNotExist: analysis = update_analysis(result) if analysis: self.stderr.write('- %s was created\n' % result['analysis_id']) else: self.stderr.write('- %s was skipped\n' % result['analysis_id']) self.stderr.write('Downloading not existent cache ...\n') self.done_count = 0 self.error_count = 0 PROCESSES = getattr(settings, 'MULTIPROCESSING_CORES', None) if not PROCESSES: PROCESSES = int(multiprocessing.cpu_count() / 2) or 1 self.pool = multiprocessing.Pool(PROCESSES) self.tasks = [] for analysis in Analysis.objects.all(): if not is_cart_cache_exists( analysis_id=analysis.analysis_id, last_modified=analysis.last_modified): self.tasks.append(( analysis.analysis_id, analysis.last_modified)) if len(self.tasks) >= PROCESSES * 5: self.process_tasks() if len(self.tasks): self.process_tasks() self.stdout.write( '---\n%d cache files were updated.\n' % self.done_count) self.stderr.write( '---\n%d cache files were updated.\n' % self.done_count) if self.error_count: raise CommandError( '%d errors occurred. You can find them in the logs.\n' % self.error_count)
def cart_add_all_files(request): # 'Add all to cart' pressed form = AllItemsForm(request.POST) if form.is_valid(): try: # calculate query raw_filters = form.cleaned_data['filters'] filters = get_filters_dict(raw_filters) q = raw_filters.get('q') queries = [] if q: # FIXME: temporary hack to work around GNOS not quoting Solr query # FIXME: this is temporary hack, need for multiple requests will be fixed at CGHub if browser_text_search.useAllMetadataIndex: query = {'all_metadata': browser_text_search.ws_query(q)} query.update(filters) queries = [query] else: query = {'xml_text': u"(%s)" % q} query.update(filters) queries = [query, {'analysis_id': q}] if not queries: queries = [filters] message_id = add_message( request, 'info', settings.ADDING_TO_CART_IN_PROGRESS_NOTIFICATION) try: with transaction.commit_on_success(): cart = Cart(request.session) for query in queries: api_request = RequestMinimal(query=query) for result in api_request.call(): cart.add(result) cart.update_stats() remove_message(request, message_id) except DatabaseError: remove_message(request, message_id) return { 'action': 'message', 'title': settings.DATABASE_ERROR_NOTIFICATION_TITLE, 'content': settings.DATABASE_ERROR_NOTIFICATION} return {'action': 'redirect', 'redirect': reverse('cart_page')} except Exception as e: cart_logger.error('Error while adding all files to cart: %s' % unicode(e)) else: cart_logger.error('AllItemsForm not valid: %s' % unicode(form.errors))
def profile_view(view): empty_database() empty_cache() time.sleep(5) request = get_request() cart = Cart(request.session) # fill cart api_request = RequestMinimal(query={'analysis_id': IDS}) for result in api_request.call(): cart.add(result) cart.update_stats() # Call view RUN_COUNT times for i in range(RUN_COUNT): print 'Run #%d ...' % (i + 1) runctx('run_view(view, request)', {'run_view': run_view}, {'view': view, 'request': request}, 'stats') p = pstats.Stats('stats') p.sort_stats('time').print_stats(20) print 'done'