def admin_cdrs_import(request):
if (request.method != 'POST'):
return render_to_response('portal/admin_cdrs.html', {})
created = 0
updated = 0
solr = Solr(settings.CDRS_SOLR_URL)
application = request.POST.get('application', '')
collection_id = request.POST.get('collection_id', '')
import_classification = request.POST.get('import_classification', '')
dt = request.POST.get('last_import_date', '')
tm = urllib.unquote(request.POST.get('last_import_time', '00:00'))
q = 'import_classifications:"' + import_classification + '"'
options = {'qt': 'forest-data'}
last_import_date = LastImportDate.get_last_import_date(dt, tm, application)
if last_import_date:
utc = last_import_date.astimezone(FixedOffset(0))
q += ' AND last_modified:[' + utc.strftime(
'%Y-%m-%dT%H:%M:%SZ') + ' TO NOW]'
try:
collections = urllib.unquote(collection_id).split(",")
for c in collections:
# Get list of datasets in each collection id
record_count = SolrUtilities().get_count_by_lastmodified(
c, import_classification, last_import_date)
retrieved = 0
while (retrieved < record_count):
to_retrieve = min(1000, record_count - retrieved)
options['collection_id'] = c
options['start'] = str(retrieved)
options['rows'] = str(to_retrieve)
results = solr.search(q, **options)
for result in results:
if 'dataset_id' in result:
if process_metadata(result):
created += 1
else:
updated += 1
retrieved = retrieved + to_retrieve
# Update the last import date
lid = LastImportDate.update_last_import_date(application)
cache.set('solr_import_date', lid.strftime('%Y-%m-%d'))
cache.set('solr_import_time', lid.strftime('%H:%M:%S'))
cache.set('solr_created', created)
cache.set('solr_updated', updated)
except Exception, e:
cache.set('solr_error', str(e))
def admin_cdrs_import(request):
if (request.method != 'POST'):
return render_to_response('portal/admin_cdrs.html', {})
created = 0
updated = 0
solr = Solr(settings.CDRS_SOLR_URL)
application = request.POST.get('application', '')
collection_id = request.POST.get('collection_id', '')
import_classification = request.POST.get('import_classification', '')
dt = request.POST.get('last_import_date', '')
tm = urllib.unquote(request.POST.get('last_import_time', '00:00'))
q = 'import_classifications:"' + import_classification + '"'
options = {'qt': 'forest-data'}
last_import_date = LastImportDate.get_last_import_date(dt, tm, application)
if last_import_date:
utc = last_import_date.astimezone(FixedOffset(0))
q += ' AND last_modified:[' + utc.strftime(
'%Y-%m-%dT%H:%M:%SZ') + ' TO NOW]'
try:
collections = urllib.unquote(collection_id).split(",")
for c in collections:
# Get list of datasets in each collection id
record_count = SolrUtilities().get_count_by_lastmodified(
c, import_classification, last_import_date)
retrieved = 0
while (retrieved < record_count):
to_retrieve = min(1000, record_count - retrieved)
options['collection_id'] = c
options['start'] = str(retrieved)
options['rows'] = str(to_retrieve)
results = solr.search(q, **options)
for result in results:
if 'dataset_id' in result:
if process_metadata(result):
created += 1
else:
updated += 1
retrieved = retrieved + to_retrieve
# Update the last import date
lid = LastImportDate.update_last_import_date(application)
cache.set('solr_import_date', lid.strftime('%Y-%m-%d'))
cache.set('solr_import_time', lid.strftime('%H:%M:%S'))
cache.set('solr_created', created)
cache.set('solr_updated', updated)
except Exception, e:
cache.set('solr_error', str(e))
def admin_cdrs_import(request):
if (request.method != 'POST'):
return render(request, 'portal/admin_cdrs.html', {})
created = 0
updated = 0
solr = Solr(settings.CDRS_SOLR_URL)
application = request.POST.get('application', '')
collection_id = request.POST.get('collection_id', '')
import_classification = request.POST.get('import_classification', '')
dt = request.POST.get('last_import_date', '')
tm = unquote(request.POST.get('last_import_time', '00:00'))
options = {'qt': 'forest-data'}
last_import_date = LastImportDate.get_last_import_date(dt, tm, application)
q = import_classification_query(import_classification, last_import_date)
try:
collections = unquote(collection_id).split(",")
for c in collections:
# Get list of datasets in each collection id
created, updated = get_collection_datasets(c, solr, options, q,
import_classification,
last_import_date,
created, updated)
# Update the last import date
lid = LastImportDate.update_last_import_date(application)
cache.set('solr_import_date', lid.strftime('%Y-%m-%d'))
cache.set('solr_import_time', lid.strftime('%H:%M:%S'))
cache.set('solr_created', created)
cache.set('solr_updated', updated)
except Exception as e:
cache.set('solr_error', str(e))
cache.set('solr_complete', True)
response = {'complete': True}
http_response = HttpResponse(json.dumps(response),
content_type='application/json')
http_response['Cache-Control'] = 'max-age=0,no-cache,no-store'
return http_response
def admin_cdrs_import(request):
if (request.method != 'POST'):
return render(request, 'portal/admin_cdrs.html', {})
created = 0
updated = 0
solr = Solr(settings.CDRS_SOLR_URL)
application = request.POST.get('application', '')
collection_id = request.POST.get('collection_id', '')
import_classification = request.POST.get('import_classification', '')
dt = request.POST.get('last_import_date', '')
tm = unquote(request.POST.get('last_import_time', '00:00'))
options = {'qt': 'forest-data'}
last_import_date = LastImportDate.get_last_import_date(dt, tm, application)
q = import_classification_query(import_classification, last_import_date)
try:
collections = unquote(collection_id).split(",")
for c in collections:
# Get list of datasets in each collection id
created, updated = get_collection_datasets(
c, solr, options, q, import_classification, last_import_date,
created, updated)
# Update the last import date
lid = LastImportDate.update_last_import_date(application)
cache.set('solr_import_date', lid.strftime('%Y-%m-%d'))
cache.set('solr_import_time', lid.strftime('%H:%M:%S'))
cache.set('solr_created', created)
cache.set('solr_updated', updated)
except Exception as e:
cache.set('solr_error', str(e))
cache.set('solr_complete', True)
response = {'complete': True}
http_response = HttpResponse(
json.dumps(response), content_type='application/json')
http_response['Cache-Control'] = 'max-age=0,no-cache,no-store'
return http_response
def previewsolr(request):
response = {}
solr = SolrUtilities()
application = request.POST.get('application', '')
collection_id = request.POST.get('collection_id', '')
import_classification = request.POST.get('import_classification', '')
dt = request.POST.get('last_import_date', '')
tm = urllib.unquote(request.POST.get('last_import_time', '00:00'))
last_import_date = LastImportDate.get_last_import_date(dt, tm, application)
response['record_count'] = solr.get_count_by_lastmodified(
collection_id, import_classification, last_import_date)
if last_import_date:
response['last_import_date'] = last_import_date.strftime('%Y-%m-%d')
response['last_import_time'] = last_import_date.strftime('%H:%M:%S')
http_response = HttpResponse(json.dumps(response),
content_type='application/json')
http_response['Cache-Control'] = 'max-age=0,no-cache,no-store'
return http_response
def previewsolr(request):
response = {}
solr = SolrUtilities()
application = request.POST.get('application', '')
collection_id = request.POST.get('collection_id', '')
import_classification = request.POST.get('import_classification', '')
dt = request.POST.get('last_import_date', '')
tm = urllib.unquote(request.POST.get('last_import_time', '00:00'))
last_import_date = LastImportDate.get_last_import_date(dt, tm, application)
response['record_count'] = solr.get_count_by_lastmodified(
collection_id, import_classification, last_import_date)
if last_import_date:
response['last_import_date'] = last_import_date.strftime('%Y-%m-%d')
response['last_import_time'] = last_import_date.strftime('%H:%M:%S')
http_response = HttpResponse(
json.dumps(response), content_type='application/json')
http_response['Cache-Control'] = 'max-age=0,no-cache,no-store'
return http_response
def test_update_last_import_date(self):
d = LastImportDate.update_last_import_date("foo")
self.assertTrue(hasattr(d, 'year'))
d2 = LastImportDate.update_last_import_date("foo")
self.assertNotEqual(d, d2)
def test_get_last_import_date_invalid(self):
d = LastImportDate.get_last_import_date("garbage", "more garbage",
"foo")
self.assertTrue(d is None)
def test_get_last_import_date(self):
d = LastImportDate.get_last_import_date("2010-01-01", "13:15:00",
"foo")
self.assertTrue(d is not None)
def loadsolr(request):
application = request.POST.get('application', '')
collection_id = request.POST.get('collection_id', '')
import_classification = request.POST.get('import_classification', '')
dt = request.POST.get('last_import_date', '')
tm = urllib.unquote(request.POST.get('last_import_time', '00:00'))
limit_records = int(request.POST.get('limit_records', '0'))
solr = Solr(settings.CDRS_SOLR_URL)
# stash the station mappings into a python map
stations = {}
for sm in StationMapping.objects.all():
stations[sm.abbreviation] = sm.station
created_count = 0
updated_count = 0
try:
cursor = connection.cursor()
next_expected_timestamp = None
last_valid_temp = None
prev_station = None
retrieved = 0
q = 'import_classifications:"' + import_classification + \
'" AND (record_subject:"Array ID 60" ' + \
'OR record_subject:"Array ID 101")'
options = {
'qt': 'forest-data',
'collection_id': collection_id,
'sort': 'latitude asc,record_datetime asc'
}
last_import_date = LastImportDate.get_last_import_date(
dt, tm, application)
if last_import_date:
utc = last_import_date.astimezone(FixedOffset(0))
q += ' AND last_modified:[' + utc.strftime(
'%Y-%m-%dT%H:%M:%SZ') + ' TO NOW]'
record_count = SolrUtilities().get_count_by_lastmodified(
collection_id, import_classification, last_import_date)
if limit_records > 0:
record_count = limit_records
while (retrieved < record_count):
to_retrieve = min(1000, record_count - retrieved)
options['start'] = str(retrieved)
options['rows'] = str(to_retrieve)
results = solr.search(q, **options)
for result in results:
temp = Decimal(str(result['temp_c_avg'])).quantize(
Decimal("0.01"), ROUND_HALF_UP)
dt = _utc_to_est(result['record_datetime'])
for x in result['import_classifications']:
if x in stations:
station = stations[x]
break
if (station and dt and temp is not None):
(next_expected_timestamp, last_valid_temp, prev_station,
created, updated) = _process_row(cursor, dt, station,
float(temp),
next_expected_timestamp,
last_valid_temp,
prev_station)
created_count = created_count + created
updated_count = updated_count + updated
retrieved = retrieved + to_retrieve
# Update the last import date
lid = LastImportDate.update_last_import_date(application)
cache.set('solr_import_date', lid.strftime('%Y-%m-%d'))
cache.set('solr_import_time', lid.strftime('%H:%M:%S'))
cache.set('solr_created', created_count)
cache.set('solr_updated', updated_count)
except Exception, e:
cache.set('solr_error', str(e))
def loadsolr(request):
application = request.POST.get('application', '')
collection_id = request.POST.get('collection_id', '')
import_classification = request.POST.get('import_classification', '')
dt = request.POST.get('last_import_date', '')
tm = unquote(request.POST.get('last_import_time', '00:00'))
limit_records = int(request.POST.get('limit_records', '0'))
solr = Solr(settings.CDRS_SOLR_URL)
stations = station_mappings_dict()
created_count = 0
updated_count = 0
try:
cursor = connection.cursor()
next_expected_timestamp = None
last_valid_temp = None
prev_station = None
retrieved = 0
last_import_date = LastImportDate.get_last_import_date(
dt, tm, application)
options = {
'qt': 'forest-data',
'collection_id': collection_id,
'sort': 'latitude asc,record_datetime asc'
}
q = import_classifications_query(import_classification,
last_import_date)
record_count = get_record_count(collection_id, import_classification,
last_import_date, limit_records)
while (retrieved < record_count):
to_retrieve = min(1000, record_count - retrieved)
options['start'] = str(retrieved)
options['rows'] = str(to_retrieve)
results = solr.search(q, **options)
for result in results:
temp = Decimal(str(result['temp_c_avg'])).quantize(
Decimal("0.01"), ROUND_HALF_UP)
dt = _utc_to_est(result['record_datetime'])
for x in result['import_classifications']:
if x in stations:
station = stations[x]
break
(created_count, updated_count, next_expected_timestamp,
last_valid_temp, prev_station) = process_station_row(
cursor, station, dt, temp, created_count, updated_count,
next_expected_timestamp, last_valid_temp, prev_station)
retrieved = retrieved + to_retrieve
# Update the last import date
lid = LastImportDate.update_last_import_date(application)
cache.set('solr_import_date', lid.strftime('%Y-%m-%d'))
cache.set('solr_import_time', lid.strftime('%H:%M:%S'))
cache.set('solr_created', created_count)
cache.set('solr_updated', updated_count)
except Exception as e:
cache.set('solr_error', str(e))
cache.set('solr_complete', True)
response = {'complete': True}
http_response = HttpResponse(dumps(response),
content_type='application/json')
http_response['Cache-Control'] = 'max-age=0,no-cache,no-store'
return http_response
def loadsolr(request):
application = request.POST.get('application', '')
collection_id = request.POST.get('collection_id', '')
import_classification = request.POST.get('import_classification', '')
dt = request.POST.get('last_import_date', '')
tm = urllib.unquote(request.POST.get('last_import_time', '00:00'))
limit_records = int(request.POST.get('limit_records', '0'))
solr = Solr(settings.CDRS_SOLR_URL)
# stash the station mappings into a python map
stations = {}
for sm in StationMapping.objects.all():
stations[sm.abbreviation] = sm.station
created_count = 0
updated_count = 0
try:
cursor = connection.cursor()
next_expected_timestamp = None
last_valid_temp = None
prev_station = None
retrieved = 0
q = 'import_classifications:"' + import_classification + \
'" AND (record_subject:"Array ID 60" ' + \
'OR record_subject:"Array ID 101")'
options = {'qt': 'forest-data',
'collection_id': collection_id,
'sort': 'latitude asc,record_datetime asc'}
last_import_date = LastImportDate.get_last_import_date(
dt, tm, application)
if last_import_date:
utc = last_import_date.astimezone(FixedOffset(0))
q += ' AND last_modified:[' + utc.strftime(
'%Y-%m-%dT%H:%M:%SZ') + ' TO NOW]'
record_count = SolrUtilities().get_count_by_lastmodified(
collection_id, import_classification, last_import_date)
if limit_records > 0:
record_count = limit_records
while (retrieved < record_count):
to_retrieve = min(1000, record_count - retrieved)
options['start'] = str(retrieved)
options['rows'] = str(to_retrieve)
results = solr.search(q, **options)
for result in results:
temp = Decimal(str(result['temp_c_avg'])
).quantize(Decimal("0.01"), ROUND_HALF_UP)
dt = _utc_to_est(result['record_datetime'])
for x in result['import_classifications']:
if x in stations:
station = stations[x]
break
if (station and dt and temp is not None):
(next_expected_timestamp,
last_valid_temp,
prev_station,
created,
updated) = _process_row(cursor, dt, station,
float(temp),
next_expected_timestamp,
last_valid_temp,
prev_station)
created_count = created_count + created
updated_count = updated_count + updated
retrieved = retrieved + to_retrieve
# Update the last import date
lid = LastImportDate.update_last_import_date(application)
cache.set('solr_import_date', lid.strftime('%Y-%m-%d'))
cache.set('solr_import_time', lid.strftime('%H:%M:%S'))
cache.set('solr_created', created_count)
cache.set('solr_updated', updated_count)
except Exception, e:
cache.set('solr_error', str(e))