def test_update_analysis(self):
# create not existent
analysis_id = '016b792f-e659-4143-b833-163141e21363'
self.assertFalse(Analysis.objects.filter(
analysis_id=analysis_id).exists())
api_request = RequestMinimal(query={'analysis_id': analysis_id})
result = api_request.call().next()
analysis = update_analysis(result)
self.assertTrue(Analysis.objects.filter(
analysis_id=analysis_id).exists())
# update
new_last_modified = '2077-77-77T77:77:77Z'
new_center_name = 'ABCD'
self.assertNotEqual(analysis.last_modified, new_last_modified)
self.assertNotEqual(analysis.center_name, new_center_name)
analysis.last_modified = new_last_modified
analysis.center_name = new_center_name
analysis.save()
self.assertEqual(analysis.last_modified, new_last_modified)
self.assertEqual(analysis.center_name, new_center_name)
analysis = update_analysis(result)
self.assertNotEqual(analysis.last_modified, new_last_modified)
self.assertNotEqual(analysis.center_name, new_center_name)
# check all attributes are setted
api_request = RequestDetailJSON(query={'analysis_id': analysis_id})
result = api_request.call().next()
for attr in CART_SORT_ATTRIBUTES:
self.assertEqual(result.get(attr), getattr(analysis, attr))
def handle(self, *args, **options):
self.stdout.write('Searching for outdated analysises ...\n')
self.stderr.write('Searching for outdated analysises ...\n')
api_request = RequestMinimal(query={})
for result in api_request.call():
try:
analysis = Analysis.objects.get(
analysis_id=result['analysis_id'],
last_modified=result['last_modified'])
if analysis.last_modified != result['last_modified']:
analysis = update_analysis(result)
if analysis:
self.stderr.write('- %s was updated\n' % result['analysis_id'])
else:
self.stderr.write('- %s was skipped\n' % result['analysis_id'])
except Analysis.DoesNotExist:
analysis = update_analysis(result)
if analysis:
self.stderr.write('- %s was created\n' % result['analysis_id'])
else:
self.stderr.write('- %s was skipped\n' % result['analysis_id'])
self.stderr.write('Downloading not existent cache ...\n')
self.done_count = 0
self.error_count = 0
PROCESSES = getattr(settings, 'MULTIPROCESSING_CORES', None)
if not PROCESSES:
PROCESSES = int(multiprocessing.cpu_count() / 2) or 1
self.pool = multiprocessing.Pool(PROCESSES)
self.tasks = []
for analysis in Analysis.objects.all():
if not is_cart_cache_exists(
analysis_id=analysis.analysis_id,
last_modified=analysis.last_modified):
self.tasks.append((
analysis.analysis_id,
analysis.last_modified))
if len(self.tasks) >= PROCESSES * 5:
self.process_tasks()
if len(self.tasks):
self.process_tasks()
self.stdout.write(
'---\n%d cache files were updated.\n' % self.done_count)
self.stderr.write(
'---\n%d cache files were updated.\n' % self.done_count)
if self.error_count:
raise CommandError(
'%d errors occurred. You can find them in the logs.\n' % self.error_count)
def cart_add_all_files(request):
# 'Add all to cart' pressed
form = AllItemsForm(request.POST)
if form.is_valid():
try:
# calculate query
raw_filters = form.cleaned_data['filters']
filters = get_filters_dict(raw_filters)
q = raw_filters.get('q')
queries = []
if q:
# FIXME: temporary hack to work around GNOS not quoting Solr query
# FIXME: this is temporary hack, need for multiple requests will be fixed at CGHub
if browser_text_search.useAllMetadataIndex:
query = {'all_metadata': browser_text_search.ws_query(q)}
query.update(filters)
queries = [query]
else:
query = {'xml_text': u"(%s)" % q}
query.update(filters)
queries = [query, {'analysis_id': q}]
if not queries:
queries = [filters]
message_id = add_message(
request, 'info',
settings.ADDING_TO_CART_IN_PROGRESS_NOTIFICATION)
try:
with transaction.commit_on_success():
cart = Cart(request.session)
for query in queries:
api_request = RequestMinimal(query=query)
for result in api_request.call():
cart.add(result)
cart.update_stats()
remove_message(request, message_id)
except DatabaseError:
remove_message(request, message_id)
return {
'action': 'message',
'title': settings.DATABASE_ERROR_NOTIFICATION_TITLE,
'content': settings.DATABASE_ERROR_NOTIFICATION}
return {'action': 'redirect', 'redirect': reverse('cart_page')}
except Exception as e:
cart_logger.error('Error while adding all files to cart: %s' % unicode(e))
else:
cart_logger.error('AllItemsForm not valid: %s' % unicode(form.errors))
def profile_view(view):
empty_database()
empty_cache()
time.sleep(5)
request = get_request()
cart = Cart(request.session)
# fill cart
api_request = RequestMinimal(query={'analysis_id': IDS})
for result in api_request.call():
cart.add(result)
cart.update_stats()
# Call view RUN_COUNT times
for i in range(RUN_COUNT):
print 'Run #%d ...' % (i + 1)
runctx('run_view(view, request)',
{'run_view': run_view},
{'view': view, 'request': request}, 'stats')
p = pstats.Stats('stats')
p.sort_stats('time').print_stats(20)
print 'done'