def create_record(author, location, date):
record = Record(location=location)
record.author = author
record.channel = Channel.objects.get(name="Proyecto Pance")
record.save()
record.updateCreationDate(date)
return record
def test_get_initial_queryset(self):
config = TestConfiguration()
other_record = Record(job_id=9348275987)
view = DTRecordsJson(kwargs={})
records = view.get_initial_queryset()
job_ids = set(map(lambda x: x.job.id, records))
self.assertNotIn(0, job_ids)
def test_report_success(self):
record = Record(student_id=self.student_id)
record.state = Record.LOCKED
record.save()
vote_token = AuthToken.generate(self.student_id, str(self.station.external_id), '70')
vote_token.save()
url = reverse('report')
data = {
'uid': self.student_id,
'vote_token': vote_token.code,
'token': self.token,
'api_key': settings.API_KEY, 'version': settings.API_VERSION,
}
response = self.client.post(url, data)
self.assertEqual(response.data, {'status': 'success'})
def __init__(self):
self.user = User.objects.create(username='******',
password='******',
is_superuser=True)
self.org = Organization.objects.create(name="Test Organization")
self.record_group = RecordGroup.objects.create(
organization=self.org, name="Test Record Group")
self.job = Job.objects.create(record_group=self.record_group,
user=self.user,
job_type="HarvestJob",
job_details='{"test_key": "test value"}',
name="Test Job")
self.downstream_job = Job.objects.create(
record_group=self.record_group,
user=self.user,
job_type="TransformJob",
job_details='{"test_key": "test value"}',
name="Test Transform Job")
JobInput.objects.create(job=self.downstream_job, input_job=self.job)
# TODO: the test framework should be clearing all the dbs, not just mysql
old_records = Record.objects(job_id=self.job.id)
for record in old_records:
record.delete()
self.record = Record.objects.create(job_id=self.job.id,
record_id='testrecord',
document=TEST_DOCUMENT)
self.job.update_record_count()
def test_complete_success(self):
record = Record(student_id=self.student_id)
record.state = Record.VOTING
record.save()
token = AuthToken.generate(self.student_id, str(self.station.external_id), '70')
token.save()
url = 'https://{0}{1}?callback={2}'.format(
settings.CALLBACK_DOMAIN, reverse('callback'), token.confirm_code)
response = self.client.get(url)
record = Record.objects.get(student_id=self.student_id)
self.assertEqual(record.state, Record.USED)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data, {
'status': 'success',
'message': 'all correct',
})
def test_confirm_success(self):
record = Record(student_id=self.student_id)
record.state = Record.LOCKED
record.save()
vote_token = AuthToken.generate(self.student_id, str(self.station.external_id), '70')
vote_token.save()
url = reverse('confirm')
data = {'uid': self.student_id,
'vote_token': vote_token.code, 'token': self.token,
'api_key': settings.API_KEY, 'version': settings.API_VERSION}
response = self.client.post(url, data)
callback = 'https://{0}{1}?callback={2}'.format(
settings.CALLBACK_DOMAIN, reverse('callback'), vote_token.confirm_code)
self.assertEqual(response.data, {
'status': 'success',
'ballot': self.authcode.code,
'callback': callback,
})
def populateWorldRecords(url, stats_type, countries_df):
print_info(f"Fetching content for stats_type [{stats_type}]..")
print_info(f"URL in use [{url}]")
with requests.Session() as s:
download = s.get(url)
decoded_content = download.content.decode('utf-8')
# local_file_name = f'datasets/{stats_type}.csv'
# print_info(f"Writing dowloaded content to file[{local_file_name}]..")
# with open(local_file_name, "w") as outfile:
# outfile.write(decoded_content)
# print_info("Writing summary to local file..Done")
rows_fetched = list(csv.reader(decoded_content.splitlines(),
delimiter=','))
print_info(f"Fetching content for stats_type [{stats_type}]..Done")
# Handle header row
header_row = rows_fetched.pop(0) # Header is the first row.
header_row.pop(0) # Remove the value 'Province/State'
header_row.pop(0) # Remove the value 'Country/Region'
header_row.pop(0) # Remove the value 'Lat'
header_row.pop(0) # Remove the value 'Long'
latest_stats_date = rectifyDateFormat(header_row[-1])
# stats_dates_csv =
stats_dates_csv = rectifyDateFormat(dates_csv=(",".join(header_row)))
print_info("Creating objects..")
objects_list = []
ignored_countries = [
'Diamond Princess', 'West Bank and Gaza', 'Kosovo', 'MS Zaandam'
]
for row in rows_fetched[:]:
state_province = row.pop(0)
country_region = row.pop(0)
if (country_region in ignored_countries):
country_alpha3 = '---'
else:
country_alpha3 = countries_df.loc[country_region, 'alpha3']
latitude = row.pop(0)
longitude = row.pop(0)
stats_type = stats_type
stats_value_csv = ",".join(row)
latest_stats_value = row[-1] or 0
# Create model Record instances
obj = Record(
state_province=state_province,
country_region=country_region,
country_alpha3=country_alpha3,
latitude=latitude,
longitude=longitude,
stats_type=stats_type,
latest_stats_date=latest_stats_date,
latest_stats_value=latest_stats_value,
stats_dates_csv=stats_dates_csv,
stats_value_csv=stats_value_csv,
)
objects_list.append(obj)
print_info("Creating objects..Done")
print_info(f"Total objects created = {len(objects_list)}")
print_info(f"Inserting records for stats_type[{stats_type}]..")
Record.objects.bulk_create(objects_list)
print_info(f"Inserting records for stats_type[{stats_type}]..Done")
return decoded_content
def populate_records_india(
url='https://api.rootnet.in/covid19-in/stats/daily'):
print_info("Handling India records..")
Record.objects.filter(country_region='India').delete()
# Globals
states_lat_long = {
"India": {
"lat": 20.5937,
"long": 78.9629
},
"Andhra Pradesh": {
"lat": 15.9129,
"long": 79.7400
},
"Assam": {
"lat": 26.244156,
"long": 92.537842
},
"Bihar": {
"lat": 25.0961,
"long": 85.3131
},
"Chandigarh": {
"lat": 30.7333,
"long": 76.7794
},
"Chhattisgarh": {
"lat": 21.295132,
"long": 81.828232
},
"Delhi": {
"lat": 28.7041,
"long": 77.1025
},
"Gujarat": {
"lat": 22.309425,
"long": 72.136230
},
"Haryana": {
"lat": 29.238478,
"long": 76.431885
},
"Himachal Pradesh": {
"lat": 32.084206,
"long": 77.571167
},
"Jammu and Kashmir": {
"lat": 33.7782,
"long": 76.5762
},
"Karnataka": {
"lat": 15.317277,
"long": 75.713890
},
"Kerala": {
"lat": 10.850516,
"long": 76.271080
},
"Ladakh": {
"lat": 34.152588,
"long": 77.577049
},
"Madhya Pradesh": {
"lat": 23.473324,
"long": 77.947998
},
"Maharashtra": {
"lat": 19.601194,
"long": 75.552979
},
"Odisha": {
"lat": 20.940920,
"long": 84.803467
},
"Puducherry": {
"lat": 11.9416,
"long": 79.8083
},
"Punjab": {
"lat": 31.1471,
"long": 75.3412
},
"Rajasthan": {
"lat": 27.391277,
"long": 73.432617
},
"Tamil Nadu": {
"lat": 11.127123,
"long": 78.656891
},
"Telangana": {
"lat": 17.123184,
"long": 79.208824
},
"Telengana": {
"lat": 17.123184,
"long": 79.208824
},
"Tripura": {
"lat": 23.745127,
"long": 91.746826
},
"Uttar Pradesh": {
"lat": 28.207609,
"long": 79.826660
},
"Uttarakhand": {
"lat": 30.0668,
"long": 79.0193
},
"West Bengal": {
"lat": 22.978624,
"long": 87.747803
}
}
state_wise_stats = {}
# Fetch JSON data from url
r = requests.get(url)
r_json = r.json()
for data in r_json['data']:
date = data['day'] # 2020-03-10
for regional in data['regional']:
state = regional['loc']
if (state in state_wise_stats.keys()):
pass
else:
state_wise_stats[state] = {}
state_wise_stats[state]['confirmed_csv'] = ''
state_wise_stats[state]['recovered_csv'] = ''
state_wise_stats[state]['deaths_csv'] = ''
state_wise_stats[state]['dates_csv'] = ''
# Handle the case where the state is not present in the states_lat_long dict
if (state in states_lat_long.keys()):
state_wise_stats[state]['lat'] = states_lat_long[state]['lat']
state_wise_stats[state]['long'] = states_lat_long[state][
'long']
else:
state_wise_stats[state]['lat'] = states_lat_long['India'][
'lat']
state_wise_stats[state]['long'] = states_lat_long['India'][
'long']
state_wise_stats[state]['recovered_csv'] = state_wise_stats[state][
'recovered_csv'] + str(regional['discharged']) + ","
state_wise_stats[state]['confirmed_csv'] = state_wise_stats[state][
'confirmed_csv'] + str(regional['confirmedCasesIndian'] +
regional['confirmedCasesForeign']) + ","
state_wise_stats[state]['deaths_csv'] = state_wise_stats[state][
'deaths_csv'] + str(regional['deaths']) + ","
state_wise_stats[state]['dates_csv'] = state_wise_stats[state][
'dates_csv'] + str(date) + ","
state_wise_stats[state]['confirmed_latest'] = regional[
'confirmedCasesIndian'] + regional['confirmedCasesForeign']
state_wise_stats[state]['recovered_latest'] = regional[
'discharged']
state_wise_stats[state]['deaths_latest'] = regional['deaths']
state_wise_stats[state]['date_latest'] = str(date)
# At this time, we have collected all the data into the state_wise_stats dict
# Next step - Load data onto database using bulk insert
# Bulk insert requires array of objects to be created
objects_list = []
for state in state_wise_stats:
# For each state, we create 3 objects - 1.Confirmed 2.Recovered 3.Deaths
obj = Record(
state_province=state,
country_region='India',
country_alpha3='IND',
latitude=state_wise_stats[state]['lat'],
longitude=state_wise_stats[state]['long'],
stats_type='confirmed',
latest_stats_date=state_wise_stats[state]['date_latest'],
latest_stats_value=state_wise_stats[state]['confirmed_latest'],
stats_dates_csv=state_wise_stats[state]['dates_csv'].rstrip(','),
stats_value_csv=state_wise_stats[state]['confirmed_csv'].rstrip(
','),
)
objects_list.append(obj)
obj = Record(
state_province=state,
country_region='India',
country_alpha3='IND',
latitude=state_wise_stats[state]['lat'],
longitude=state_wise_stats[state]['long'],
stats_type='deaths',
latest_stats_date=state_wise_stats[state]['date_latest'],
latest_stats_value=state_wise_stats[state]['deaths_latest'],
stats_dates_csv=state_wise_stats[state]['dates_csv'].rstrip(','),
stats_value_csv=state_wise_stats[state]['deaths_csv'].rstrip(','),
)
objects_list.append(obj)
obj = Record(
latitude=state_wise_stats[state]['lat'],
longitude=state_wise_stats[state]['long'],
stats_type='recovered',
state_province=state,
country_region='India',
country_alpha3='IND',
stats_dates_csv=state_wise_stats[state]['dates_csv'].rstrip(','),
stats_value_csv=state_wise_stats[state]['recovered_csv'].rstrip(
','),
latest_stats_date=state_wise_stats[state]['date_latest'],
latest_stats_value=state_wise_stats[state]['recovered_latest'])
objects_list.append(obj)
print_info("Inserting INDIA records..")
Record.objects.bulk_create(objects_list)
print_info("Inserting INDIA records..Done")
print_info("Handling India records..Done")
def authenticate(request):
# Check parameters
internal_id = request.data['cid']
raw_student_id = request.data['uid']
station_id = request.station
if settings.ENFORCE_CARD_VALIDATION:
# Parse student ID
if re.match(r'[A-Z]\d{2}[0-9A-Z]\d{6}', raw_student_id) and re.match(r'[0-9a-f]{8}', internal_id) and re.match(r'\d+', station_id):
# Extract parameters
student_id = raw_student_id[:-1]
revision = int(raw_student_id[-1:])
logger.info('Station %s request for card %s[%s]', station_id, student_id, revision)
else:
# Malformed card information
logger.info('Station %s request for card %s (%s)', station_id, raw_student_id, internal_id)
return error('card_invalid')
else:
# Do not reveal full internal ID as ACA requested
logger.info('Station %s request for card (%s****)', station_id, internal_id[:4])
# Call ACA API
try:
aca_info = service.to_student_id(internal_id)
except URLError:
logger.exception('Failed to connect to ACA server')
return error('external_error', status.HTTP_502_BAD_GATEWAY)
except service.ExternalError as e:
if not settings.ENFORCE_CARD_VALIDATION:
# We can only reveal full internal ID if it’s an invalid card
logger.exception('Card rejected by ACA server (%s), reason %s', internal_id, e.reason)
else:
logger.exception('Card rejected by ACA server, reason %s', e.reason)
# Tell clients the exact reason of error
if e.reason == 'card_invalid' or e.reason == 'student_not_found':
return error('card_invalid')
elif e.reason == 'card_blacklisted':
return error('card_suspicious')
return error('external_error', status.HTTP_502_BAD_GATEWAY)
else:
if settings.ENFORCE_CARD_VALIDATION:
student_id = aca_info.id
revision = 0
logger.info('User %s (%s) checked', student_id, aca_info.type)
elif aca_info.id != student_id:
logger.info('ID %s returned instead', aca_info.id)
return error('card_suspicious')
# Check vote record
try:
record = Record.objects.get(student_id=student_id)
if settings.ENFORCE_CARD_VALIDATION and record.revision != revision:
# ACA claim the card valid!
logger.info('Expect revision %s, recorded %s', revision, record.revision)
return error('card_suspicious')
if record.state == Record.VOTING:
# Automaticlly unlock stuck record
record.state = Record.AVAILABLE
record.save()
logger.info('Reset %s state from VOTING', student_id)
if record.state != Record.AVAILABLE:
logger.error('Duplicate entry (%s)', student_id)
return error('duplicate_entry')
except Record.DoesNotExist:
record = Record(student_id=student_id, revision=revision)
# Build up kind identifier
try:
college = settings.COLLEGE_IDS[aca_info.college]
except KeyError:
# Use student ID as an alternative
logger.warning('No matching college for ACA entry %s', aca_info.college)
college = student_id[3]
# In rare cases, we may encounter students without colleges
if college not in settings.COLLEGE_NAMES:
logger.warning('No matching college for ID %s', college)
college = '0'
# Determine graduate status
type_code = student_id[0]
kind = college
try:
override = OverrideEntry.objects.get(student_id=student_id)
kind = override.kind
except OverrideEntry.DoesNotExist:
if type_code in settings.GRADUATE_CODES:
if aca_info.department in settings.JOINT_DEPARTMENT_CODES:
kind += 'B'
else:
kind += '1'
elif type_code in settings.UNDERGRADUATE_CODES:
if aca_info.department in settings.JOINT_DEPARTMENT_CODES:
kind += 'A'
else:
kind += '0'
# Check if student has eligible identity
if kind not in settings.KINDS:
return error('unqualified')
# Generate record and token
record.state = Record.LOCKED
record.save()
token = AuthToken.generate(student_id, station_id, kind)
token.save()
logger.info('Auth token issued: %s', token.code)
return Response({'status': 'success', 'uid': student_id, 'type': settings.KINDS[kind], 'vote_token': token.code})
def authenticate(request):
# Check parameters
internal_id = request.data['cid']
raw_student_id = request.data['uid']
station_id = request.station
if settings.ENFORCE_CARD_VALIDATION:
# Parse student ID
if re.match(r'[A-Z]\d{2}[0-9A-Z]\d{6}', raw_student_id) and re.match(r'[0-9a-f]{8}', internal_id) and re.match(r'\d+', station_id):
# Extract parameters
student_id = raw_student_id[:-1]
revision = int(raw_student_id[-1:])
logger.info('Station %s request for card %s[%s]', station_id, student_id, revision)
else:
# Malformed card information
logger.info('Station %s request for card %s (%s)', station_id, raw_student_id, internal_id)
return error('card_invalid')
else:
# Do not reveal full internal ID as ACA requested
logger.info('Station %s request for card (%s****)', station_id, internal_id[:4])
# Call ACA API
try:
aca_info = service.to_student_id(internal_id)
except URLError:
logger.exception('Failed to connect to ACA server')
return error('external_error', status.HTTP_502_BAD_GATEWAY)
except service.ExternalError as e:
if not settings.ENFORCE_CARD_VALIDATION:
# We can only reveal full internal ID if it’s an invalid card
logger.exception('Card rejected by ACA server (%s), reason %s', internal_id, e.reason)
else:
logger.exception('Card rejected by ACA server, reason %s', e.reason)
# Tell clients the exact reason of error
if e.reason == 'card_invalid' or e.reason == 'student_not_found':
return error('card_invalid')
elif e.reason == 'card_blacklisted':
return error('card_suspicious')
return error('external_error', status.HTTP_502_BAD_GATEWAY)
else:
if not settings.ENFORCE_CARD_VALIDATION:
student_id = aca_info.id
revision = 0
logger.info('User %s (%s) checked', student_id, aca_info.type)
elif aca_info.id != student_id:
logger.info('ID %s returned instead', aca_info.id)
return error('card_suspicious')
# Check vote record
try:
record = Record.objects.get(student_id=student_id)
if settings.ENFORCE_CARD_VALIDATION and record.revision != revision:
# ACA claim the card valid!
logger.info('Expect revision %s, recorded %s', revision, record.revision)
return error('card_suspicious')
if record.state == Record.VOTING:
# Automaticlly unlock stuck record
record.state = Record.AVAILABLE
record.save()
logger.info('Reset %s state from VOTING', student_id)
if record.state != Record.AVAILABLE:
logger.error('Duplicate entry (%s)', student_id)
return error('duplicate_entry')
except Record.DoesNotExist:
record = Record(student_id=student_id, revision=revision)
# Determine graduate status
kind = kind_classifier.get_kind(aca_info)
if kind is None:
return error('unqualified')
# Generate record and token
record.state = Record.LOCKED
record.save()
token = AuthToken.generate(student_id, station_id, kind)
token.save()
logger.info('Auth token issued: %s', token.code)
return Response({'status': 'success', 'uid': student_id, 'type': aca_info.college,
'college': settings.DPTCODE_NAME[aca_info.department], 'vote_token': token.code})
def job_details(request, org_id, record_group_id, job_id):
LOGGER.debug('details for job id: %s', job_id)
# get CombineJob
cjob = CombineJob.get_combine_job(job_id)
# update status
cjob.job.update_status()
# detailed record count
record_count_details = cjob.job.get_detailed_job_record_count()
# get job lineage
job_lineage = cjob.job.get_lineage()
# get dpla_bulk_data_match
dpla_bulk_data_matches = cjob.job.get_dpla_bulk_data_matches()
# check if limiting to one, pre-existing record
get_q = request.GET.get('q', None)
# job details and job type specific augment
job_detail = cjob.job.job_details_dict
# mapped field analysis, generate if not part of job_details
if 'mapped_field_analysis' in job_detail.keys():
field_counts = job_detail['mapped_field_analysis']
else:
if cjob.job.finished:
field_counts = cjob.count_indexed_fields()
cjob.job.update_job_details(
{'mapped_field_analysis': field_counts}, save=True)
else:
LOGGER.debug('job not finished, not setting')
field_counts = {}
# TODO: What is this accomplishing?
# OAI Harvest
if isinstance(cjob, HarvestOAIJob):
pass
# Static Harvest
elif isinstance(cjob, HarvestStaticXMLJob):
pass
# Transform
elif isinstance(cjob, TransformJob):
pass
# Merge/Duplicate
elif isinstance(cjob, MergeJob):
pass
# Analysis
elif isinstance(cjob, AnalysisJob):
pass
# get published records, primarily for published sets
pub_records = PublishedRecords()
oai_sets = Record.objects(job_id=cjob.job.id).item_frequencies(field='oai_set')
# get published subsets with PublishedRecords static method
published_subsets = PublishedRecords.get_subsets()
# loop through subsets and enrich
for _ in published_subsets:
# add counts
counts = mc_handle.combine.misc.find_one(
{'_id': 'published_field_counts_%s' % _['name']})
# if counts not yet calculated, do now
if counts is None:
counts = PublishedRecords(
subset=_['name']).count_indexed_fields()
_['counts'] = counts
# get field mappers
field_mappers = FieldMapper.objects.all()
# return
return render(request, 'core/job_details.html', {
'cjob': cjob,
'record_group': cjob.job.record_group,
'record_count_details': record_count_details,
'field_counts': field_counts,
'field_mappers': field_mappers,
'xml2kvp_handle': xml2kvp.XML2kvp(),
'job_lineage_json': json.dumps(job_lineage),
'dpla_bulk_data_matches': dpla_bulk_data_matches,
'q': get_q,
'job_details': job_detail,
'pr': pub_records,
'published_subsets': published_subsets,
'es_index_str': cjob.esi.es_index_str,
'breadcrumbs': breadcrumb_parser(request),
'oai_sets': dict(oai_sets)
})
def test_record_with_nonexistent_job(self):
record = Record(job_id=2345789)
self.assertEqual(record.job.name, 'No Job')
self.assertEqual(record.job.record_group.name, 'No Record Group')
