def save(self):
"""
Utilize the Database class to save this object
"""
database = Database(DB_HOST, DB_PORT)
database.set(WORKER_HEADER, self)
worker_logger.debug("Saving worker:{}".format(self.worker_id))
def save(self):
"""
Utilize the Database class to save this object
"""
database = Database(DB_HOST, DB_PORT)
database.set(SERVER_HEADER, self)
opencrowd_logger.debug("Saving opencrowd Server")
def update(self):
"""
Update this server's projects
"""
database = Database(DB_HOST, DB_PORT)
for project_id in self.project_ids:
project = database.get(PROJECT_HEADER + project_id)
project.update(database=database)
def submit_tasks(self):
"""
Will recursively enter projects saved, their tasks, and their tasks' HITs that have not been submitted yet, and
have each HIT submit itself.
"""
database = Database(DB_HOST, DB_PORT)
for project_id in self.project_ids:
project = database.get(PROJECT_HEADER + project_id)
project.submit_tasks()
def __get_questions(self):
"""
Convert question ids in self to question objects
:return: list of Question objects
:rtype: list[Question]
"""
database = Database(DB_HOST, DB_PORT)
return [database.get(QUESTION_HEADER + question_id) for question_id in self.question_ids]
def worker_exists(worker_id):
"""
Check if a worker exists
:param worker_id: id of the worker to check
:type worker_id: str
:returns: True or False
:rtype: bool
"""
database = Database(DB_HOST, DB_PORT)
return database.exists(WORKER_HEADER + worker_id)
def list_section_ids(self, project_ids=None):
"""
List all section ids
"""
database = Database(DB_HOST, DB_PORT)
section_ids = []
for question_id in self.list_question_ids(project_ids):
question = database.get(QUESTION_HEADER + question_id)
for section in question.sections:
section_ids.append(section.opencrowd_id)
return section_ids
def list_question_ids(self, project_ids=None):
"""
List all question ids
"""
database = Database(DB_HOST, DB_PORT)
question_ids = []
for HIT_id in self.list_HIT_ids(project_ids):
HIT = database.get(HIT_HEADER + HIT_id)
for question_id in HIT.question_ids:
question_ids.append(question_id)
return question_ids
def list_assignment_ids(self, project_ids=None):
"""
List all assignment ids
"""
database = Database(DB_HOST, DB_PORT)
assignment_ids = []
for HIT_id in self.list_HIT_ids(project_ids):
HIT = database.get(HIT_HEADER + HIT_id)
for assignment_id in HIT.assignment_ids:
assignment_ids.append(assignment_id)
return assignment_ids
def list_HIT_ids(self, project_ids=None):
"""
List all HIT ids
"""
database = Database(DB_HOST, DB_PORT)
HIT_ids = []
for task_id in self.list_task_ids(project_ids):
task = database.get(TASK_HEADER + task_id)
for HIT_id in task.HIT_ids:
HIT_ids.append(HIT_id)
return HIT_ids
def list_task_ids(self, project_ids=None):
"""
List all task ids
"""
database = Database(DB_HOST, DB_PORT)
task_ids = []
for project_id in self.list_project_ids(project_ids):
project = database.get(PROJECT_HEADER + project_id)
for task_id in project.task_ids:
task_ids.append(task_id)
return task_ids
def submit(self, AMT):
"""
submit the HIT through the AMT connection
:param AMT:
"""
database = Database(DB_HOST, DB_PORT)
for idx, HIT_id in enumerate(self.HIT_ids):
HIT = database.get(HIT_HEADER + HIT_id)
HIT = HIT.submit(AMT)
self.HIT_ids[idx] = HIT.HITId
self.save()
def regenerate():
"""
Restores a default opencrowd instance and the project ids it tracks. After an Opencrowd server instance is
created, it's generally best to regenerate it unless you specified particular project ids to track.
"""
database = Database(DB_HOST, DB_PORT)
if database.database.exists(SERVER_HEADER):
opencrowd_logger.debug('Loading opencrowd server...')
regenerated_server = database.get('opencrowd_server')
return regenerated_server
else:
opencrowd_logger.debug('Generating new opencrowd server...')
opencrowd_server = Opencrowd()
return opencrowd_server
def generate_dags(self):
database = Database(DB_HOST, DB_PORT)
dags = []
for question in self.questions:
dag = {}
for section in question.sections:
dag[section.opencrowd_id] = section.to_json()
if section.parents == "root":
dag["root"] = section.opencrowd_id
dag['question_id'] = question.opencrowd_id
if section.to_json()['type'] == "InputGroup":
for option_id in section.option_ids:
option = database.get(SECTION_HEADER + option_id)
dag[option.opencrowd_id] = option.to_json()
dags.append(dag)
return json.dumps(dags, indent=4)
def __generate_javascript_variable(self):
javascript_variables = []
questions = []
database = Database(DB_HOST, DB_PORT)
for question_id in self.question_ids:
questions.append(database.get(QUESTION_HEADER + question_id))
for question in questions:
javascript_variable = {}
for section in question.sections:
if section.get_name() not in javascript_variable:
javascript_variable[section.get_name()] = []
javascript_variable[section.get_name()].append(section.to_json())
# print "Section: " + section.get_name() + " : " + json.dumps(section.to_json(), indent=4)
javascript_variables.append(javascript_variable)
return json.dumps(javascript_variables, indent=4)
def worker_delete(worker_id):
"""
Delete a worker if it exists
:param worker_id: id of the worker to check
:type worker_id: str
:returns: True or False
:rtype: bool
"""
database = Database(DB_HOST, DB_PORT)
try:
if database.exists(WORKER_HEADER + worker_id):
database.delete(WORKER_HEADER + worker_id)
except Exception as e:
manager_logger.error("Unable to delete worker '{}': {}".format(
worker_id, str(e)))
def submit(self, AMT):
"""
Submit self through the AMT connection and update self with the results
:param AMT: The open AMT connection object
:return:
"""
response = AMT.push(self)
database = Database(DB_HOST, DB_PORT)
database.delete(HIT_HEADER + self.opencrowd_id)
for key in response['HIT']:
setattr(self, key, (response['HIT'][key]))
setattr(self, 'CreationTime', str(self.CreationTime))
setattr(self, 'Expiration', str(self.Expiration))
setattr(self, 'opencrowd_id', str(self.HITId))
setattr(self, 'HITId', str(self.HITId))
self.save()
return self
def get_assignment_objects(self, assignment_ids=None, project_ids=None):
"""
Return already saved assignment objects (does not run update())
:param assignment_ids: optionally specify which assignments to return based on a list of ids
:type assignment_ids: list of str
:param project_ids: optionally specify which projects to scan for assignments
:type project_ids: list of str
:returns: list of assignment objects
"""
assignment_ids = assignment_ids if assignment_ids is not None else self.list_assignment_ids(
project_ids=project_ids)
database = Database(DB_HOST, DB_PORT)
assignments = []
assignment_ids = [
assignment_ids
] if type(assignment_ids) is not list else assignment_ids
for assignment_id in assignment_ids:
assignments.append(database.get(ASSIGNMENT_HEADER + assignment_id))
return assignments
def update(self, generated_crowdsource, database=None):
"""
Update this tasks's HITs
:param generated_crowdsource: the crowdsource specifications. see opencrowd/config/opencrowd
:param database: database connection, defaults to redis in opencrowd/config/redis
"""
database = database if database is not None else Database(
DB_HOST, DB_PORT)
for HIT_id in self.HIT_ids:
HIT = database.get(HIT_HEADER + HIT_id)
HIT.update(generated_crowdsource, database)
self.save()
def update(self, generated_crowdsource, database=None):
database = database if database is not None else Database(DB_HOST, DB_PORT)
try:
assignments = generated_crowdsource.list_assignments_for_hit(self.HITId)
for assignment in assignments:
new_assignment = Assignment(assignment)
if new_assignment.AssignmentId not in self.assignment_ids:
self.assignment_ids.append(new_assignment.AssignmentId)
Manager.process_assignment(new_assignment)
self.save()
HIT_logger.info("Assignment found: {}".format(new_assignment.AssignmentId))
except Exception as e:
print(e.message)
exit(1)
def recursive_delete(self):
"""
Delete all instances, via ids, tracked on this instance
"""
project_ids = self.list_project_ids()
task_ids = self.list_task_ids()
HIT_ids = self.list_HIT_ids()
assignment_ids = self.list_assignment_ids()
question_ids = self.list_question_ids()
database = Database(DB_HOST, DB_PORT)
for question_id in question_ids:
database.delete(QUESTION_HEADER + question_id)
for assignment_id in assignment_ids:
database.delete(ASSIGNMENT_HEADER + assignment_id)
for HIT_id in HIT_ids:
database.delete(HIT_HEADER + HIT_id)
for task_id in task_ids:
database.delete(TASK_HEADER + task_id)
for project_id in project_ids:
database.delete(PROJECT_HEADER + project_id)
self.project_ids = []
self.save()
def save(self):
assignment_logger.debug("Saving Assignment {}".format(self.AssignmentId))
database = Database(DB_HOST, DB_PORT)
database.set(ASSIGNMENT_HEADER + self.AssignmentId, self)
def save(self):
question_logger.debug("Saving Question {}".format(self.opencrowd_id))
database = Database(DB_HOST, DB_PORT)
database.set(QUESTION_HEADER + self.opencrowd_id, self)
def create_HITs(self,
questions_per_assignment,
assignments_per_HIT,
method='random'):
"""
Generate [HIT]s from this task's question pool. Currently only the 'random' method
is supported.
:param questions_per_assignment: how many questions to place in each HIT.
:type questions_per_assignment: int
:param assignments_per_HIT: # workers to find for this HIT
:type assignments_per_HIT: int
:param method: 'random'
:param type: str
"""
if not isinstance(questions_per_assignment, int):
raise TypeError("'questions_per_assignment' not set")
if not isinstance(assignments_per_HIT, int):
raise TypeError("'assignments_per_HIT' not set")
# assert self.project_id is not None
assignments = []
if method == 'random':
random.shuffle(self.question_ids)
database = Database(DB_HOST, DB_PORT)
questions = [
database.get(QUESTION_HEADER + question_id)
for question_id in self.question_ids
]
assignments = [
questions[i:i + questions_per_assignment]
for i in range(0, len(questions), questions_per_assignment)
]
for idx, assignment in enumerate(assignments):
task_logger.info('{}/{}'.format(idx + 1, len(assignments)))
try:
new_hit = HIT(assignments_per_hit=assignments_per_HIT,
AutoApprovalDelayInSeconds=604800,
lifetime_in_seconds=604800,
assignment_duration_in_seconds=1800,
Reward='0.01',
Title='alpha',
Keywords='trulia, alpha, arch',
Description='alpha description',
task_id=self.opencrowd_id)
for question in assignment:
new_hit.add_question(question)
task_logger.debug('HIT {} attached question {}'.format(
new_hit.opencrowd_id, question.opencrowd_id))
# add instructions
for instruction_id in self.instruction_ids:
new_hit.add_instruction_id(instruction_id)
self.HIT_ids.append(new_hit.opencrowd_id)
self.save()
except Exception as e:
task_logger.error(
"Unable to generate HIT. An exception of type {} occurred. Arguments:\n{!r}.\nTraceback: {}"
.format(type(e).__name__, e.args, traceback.format_exc()))
def get(task_id):
task_logger.debug("Fetching Task {}".format(task_id))
database = Database(DB_HOST, DB_PORT)
return database.get(TASK_HEADER + task_id)
def save(self):
task_logger.debug("Saving Task {}".format(self.opencrowd_id))
database = Database(DB_HOST, DB_PORT)
database.set(TASK_HEADER + self.opencrowd_id, self)
def save(self):
HIT_logger.debug("Saving HIT {}".format(self.opencrowd_id))
database = Database(DB_HOST, DB_PORT)
database.set(HIT_HEADER + self.opencrowd_id, self)
def save(self):
database = Database(DB_HOST, DB_PORT)
database.set(SECTION_HEADER + self.opencrowd_id, self)
from opencrowd.model.base.task import Task
from opencrowd.model.base.HIT import HIT_HEADER
from opencrowd.model.database import Database
from opencrowd.config.redis import DB_PORT, DB_HOST
task = Task()
# create an instruction
instruction = QuestionAsInstruction()
image = Image(urls=['http://lorempixel.com/400/200/nature/'])
instruction.add_section(image)
print(image.opencrowd_id)
# create a question
question = Question()
textbox_a = TextBox(main_title='Title_a', text=['Paragraph 1a', 'Paragraph 2a'])
question.add_section(textbox_a, parents='root')
# add to task
task.add_question(question)
task.add_instruction(instruction)
# create HITs
task.create_HITs(1, 1)
# render HITs
database = Database(DB_HOST, DB_PORT)
for idx, HIT_id in enumerate(task.HIT_ids):
HIT = database.get(HIT_HEADER + HIT_id)
HIT = HIT.render('pass')
with open('instruction_render.html', 'w+') as f:
f.write(HIT)