def generate(request):
"""Reset system."""
oErr = ErrHandle()
oData = dict(status="fail")
try:
# Only allow POST command of the Su
if request.is_ajax() and request.method == "POST" and \
user_is_authenticated(request) and \
user_is_superuser(request):
# Get the parameters passed on
qd = request.POST
action = qd.get("action", "")
if action == "random":
# Remove previous data
Result.objects.all().delete()
# Make sure each woorduser is assigned to a qset
for wusr in WoordUser.objects.all():
qset = wusr.woorduserqsets.first()
if qset == None:
qset = Qset.objects.filter(woorduser__isnull=True).first()
qset.woorduser = wusr
qset.save()
# Generate random data
choices = [1, 5]
for choice in choices:
# Iterate over all questions for this particular choice
with transaction.atomic():
for idx, oQuestion in enumerate(Question.objects.filter(choice=choice).values('id')):
que_id = oQuestion['id']
oErr.Status("choice = {}, q = {}".format(choice, idx+1))
# Iterate over all users
for wusr in WoordUser.objects.all():
qset = wusr.woorduserqsets.first()
judgment = random.randint(1,10)
# 90% of the time dontknown is false
dontknow = (random.randint(1,10) == 10)
# Add the random result
obj = Result.objects.create(question_id=que_id, user=wusr, judgment=judgment, dontknown=dontknow)
# Signal that this result has been accomplished
obj = QuestionSet.objects.filter(qset=qset, question_id=que_id).first()
obj.set_status("done")
# REturn positively
oData['status'] = "ok"
else:
oData['msg'] = "Not authenticated"
except:
msg = oErr.get_error_message()
oErr.DoError("question")
oData['status'] = "error"
oData['msg'] = msg
mimetype = "application/json"
data = json.dumps(oData)
return HttpResponse(data, mimetype)
def initialize_woord(additional=None, randomize=None):
"""Fill the application if this is needed
Initial: only use stimuli.txt
Issue #147: read stimuli2021.txt, omitting first line with column headers
"""
def get_stimulus(row):
woord = ""
cat = ""
note = ""
oBack = {}
try:
if len(row) == 3:
woord = row[0]
cat = row[1]
note = row[2]
oBack = dict(woord=woord, cat=cat, note=note)
elif row != "":
# parts = re.split(r'\s+', row)
parts = re.split(r'\t', row)
if len(parts) > 1:
woord = parts[0]
cat = parts[1] # parts[1].replace("(", "").replace(")", "")
note = parts[2]
oBack = dict(woord=woord, cat=cat, note=note)
else:
# Empty row
iStop = 1
except:
msg = oErr.get_error_message()
oErr.DoError("get_stimulus")
return oBack
oErr = ErrHandle()
bNeedRandomization = False
msg = "no changes"
bOrderStimuli = True # See issue #151: (valid) choices must be treated one by one for the user
STIMULI_NAME = "stimuli2021.txt" # Was: stimuli.txt
lhtml = []
try:
# Figure out directory and file names
woord_dir = os.path.abspath(os.path.join(WRITABLE_DIR, "../woord"))
choices_file = os.path.abspath(os.path.join(woord_dir, "choices.txt"))
stimuli_file = os.path.abspath(os.path.join(woord_dir, STIMULI_NAME))
if randomize != None and randomize == "reset":
bNeedRandomization = True
# Check on number of users
if WoordUser.objects.count() <= 1:
# Create 20 random users
with transaction.atomic():
for number in range(NUMBER_OF_PARTICIPANTS):
username = WoordUser.generate_new()
WoordUser.objects.create(name=username)
# Reset done status for QuestionSet objects
with transaction.atomic():
for obj in QuestionSet.objects.filter(status="done"):
obj.status = "created"
obj.save()
# Message
lhtml.append("Created {} users".format(NUMBER_OF_PARTICIPANTS))
# Check on choices
if Choice.objects.count() == 0:
# Load choices
with open(choices_file) as f:
reader = csv.reader(f, dialect='excel', delimiter='\t')
left, right = zip(*reader)
# Make lists
left_lst = list(left)
right_lst = list(right)
# Copy and create from these lists
break_points = ['heel', 'niet']
with transaction.atomic():
for idx, left in enumerate(left_lst):
right = right_lst[idx]
name = left
for bp in break_points:
if bp in left:
name = left.split(bp)[1].strip()
break
# Create object
Choice.objects.create(name=name, left=left, right=right)
lhtml.append("Created {} choices".format(len(left_lst)))
# Check on stimuli
oErr.Status("initialize_woord #1")
if additional != None and additional == "stimuli":
# THis is a POST method, continue
count = Stimulus.objects.count()
# Delete
Stimulus.objects.all().delete()
# Prepare message
lhtml.append("Stimuli deleted: {}".format(count))
if Stimulus.objects.count() == 0:
bNeedRandomization = True
oErr.Status("initialize_woord #2")
# Load the stimuli
with open(stimuli_file) as f:
reader = csv.reader(f, dialect='excel-tab')
lst_stimuli = [get_stimulus(row) for row in reader]
oErr.Status("initialize_woord #3")
# Add the stimuli into the objects
with transaction.atomic():
# Note: skip the first line containing column headers
for oStimulus in lst_stimuli[1:]:
woord = oStimulus['woord']
cat = oStimulus['cat']
note = oStimulus['note']
Stimulus.objects.create(woord=woord, category=cat, note=note)
oErr.Status("initialize_woord #4")
lhtml.append("Read from file {} stimuli".format(len(lst_stimuli)))
# Otherwise in need of randomization?
if Question.objects.count() == 0:
bNeedRandomization = True
# Do we need to randomize?
if bNeedRandomization:
# Remove previous questions
Question.objects.all().delete()
# Remove previous questions
Qset.objects.all().delete()
lhtml.append("Removed previous questions and qsets")
# Combine stimuli and choices so that we have the full set
if bOrderStimuli:
# See issue #151
result_sets = []
with transaction.atomic():
# Iterate over the choices in the correct order
for choice in Choice.objects.filter(valid=True).order_by("id"):
# Stimuli for this result set
result = []
for stimulus in Stimulus.objects.all():
obj = Question.objects.create(stimulus=stimulus, choice=choice)
result.append(obj.id)
# Add this set to the result_sets
result_sets.append(result)
lhtml.append("Created {} questions".format(Question.objects.count()))
# Divide the shuffled questions over sets
num_sets = NUMBER_OF_PARTICIPANTS + 10
for i in range(num_sets):
# Show where we are
oErr.Status("initialize_woord random set #{}".format(i))
# Create a Qset
qset = Qset.objects.create()
# Walk all the result sets
order = 0
for result in result_sets:
# Shuffle the question object id's
random.shuffle(result)
# Make the links for this shuffle
with transaction.atomic():
for idx, question_id in enumerate(result):
order = order + 1
QuestionSet.objects.create(qset=qset, question_id=question_id, order=order)
lhtml.append("Created {} question sets".format(num_sets))
else:
result = []
# Full randomization
with transaction.atomic():
# Iterate over all stimuli (will be 5000)
# 2021: these are now 1823
for stimulus in Stimulus.objects.all():
# And then over all choices (should be just 1)
# 2021: this is now 2
for choice in Choice.objects.filter(valid=True):
# This yields 10,000 objects
obj = Question.objects.create(stimulus=stimulus, choice=choice)
result.append(obj.id)
lhtml.append("Created {} questions".format(Question.objects.count()))
# Divide the shuffled questions over sets
num_sets = NUMBER_OF_PARTICIPANTS + 10
for i in range(num_sets):
# Show where we are
oErr.Status("initialize_woord random set #{}".format(i))
# Shuffle the question object id's
random.shuffle(result)
# Create a Qset
qset = Qset.objects.create()
# Make the links for this shuffle
with transaction.atomic():
for idx, question_id in enumerate(result):
order = idx + 1
QuestionSet.objects.create(qset=qset, question_id=question_id, order=order)
lhtml.append("Created {} question sets".format(num_sets))
# COmbine the message
msg = "<br />".join(lhtml)
except:
msg = oErr.get_error_message()
oErr.DoError("initialize_woord")
return msg
def get_crpp_texts(sLng, sPart, sFormat, status):
"""Read the list of texts from the /crpp service (if available)"""
oErr = ErrHandle()
try:
# Construct the object we pass along
oTxtList = {'userid': "erwin",
'lng': sLng,
'ext': sFormat}
# Possibly add 'dir'
if sPart != None and sPart != "":
oTxtList['dir'] = sPart
# Set the correct URL
url = CRPP_HOME + '/txtlist?' + json.dumps(oTxtList)
# Default reply
oBack = {}
# Get the data from the CRPP api
try:
r = requests.get(url)
except:
# Getting an exception here probably means that the back-end is not reachable (down)
oBack['status'] = 'error'
oBack['code'] = "get_crpp_texts(): The back-end server (crpp) cannot be reached. Is it running? \n{} \nURL={}".format(
get_exc_message(), url)
return oBack
# Action depends on what we receive
if r.status_code == 200:
# Convert to JSON
reply = json.loads(r.text.replace("\t", " "))
# Get the [content] part (note: no final 's')
oContent = reply['content']
# If all is well, then we receive just a jobid
if "jobid" in oContent:
sJobId = oContent['jobid']
# Now continue to ask for the status
oTxtListStatus = {'userid': "erwin", 'jobid': sJobId}
url = CRPP_HOME + '/statusxl?' + json.dumps(oTxtListStatus)
bDone = False
while not bDone:
# Get the data from the CRPP api
try:
r = requests.get(url)
except:
error_info = sys.exc_info()
iStop = True
# Action depends on what we receive
if r.status_code == 200:
# Convert the reply to JSON
reply = json.loads(r.text.replace("\t", " "))
# Get the [content] part (note: no final 's')
oContent = reply['content']
# Get the status part
oStatus = reply['status']
if oStatus['code'] == "error":
# There is an error
oBack['status'] = 'error'
oBack['code'] = oContent['code'] + oContent['msg']
bDone = True
# Need to store this status!!!
status.set("crpp", oBack)
elif oStatus['code'] == "finished":
# The process is ready
bDone = True
# Get the textlist
oTextList = oContent['textlist']
# Define the lists
oBack['count'] = oTextList['texts']
oBack['subtype'] = oTextList['subtype']
oBack['genre'] = oTextList['genre']
oBack['paths'] = oTextList['paths']
oBack['txtlist'] = oTextList['list']
oBack['status'] = 'ok'
# What we show in the end is not what needs to be returned
oShow = dict(count=oTextList['texts'],
subtype=oTextList['subtype'],
paths=oTextList['paths'])
oShow['status'] = 'ok'
oShow['status.code'] = oStatus['code']
# Need to store this status!!!
status.set("crpp", oShow)
else:
# Update the synchronisation object that contains all relevant information
oBack['lng'] = sLng
oBack['part'] = sPart
oBack['format'] = sFormat
oBack['count'] = oContent['total']
oBack['status.code'] = oStatus['code']
oBack['last.url'] = url
status.set("crpp", oBack)
# DEBUGGING
sMsg = oStatus['code']
if 'msg' in oContent:
sMsg += " " + oContent['msg']
if 'msg' in oStatus:
sMsg += " " + oStatus['msg']
oErr.Status(sMsg)
# Make sure we wait some time before making the next request
period = 0.400
nexttime = time.time() + period
bReady = False
while not bReady:
now = time.time()
bReady = (now > nexttime)
else:
# There is an error
oBack['status'] = 'error'
oBack['code'] = r.status_code
bDone = True
else:
oBack['status'] = 'error'
oBack['code'] = ""
else:
oBack['status'] = 'error'
oBack['code'] = r.status_code
except:
oBack['status'] = 'error'
oBack['code'] = sys.exc_info()[1]
# REturn what we have
return oBack