def get_random_sequence(tag, length=None, reset=False):
_random = get_object('sequencer-random', (lambda: random.Random()))
is_old = ((lambda x:reset) if reset else None)
if length is not None:
new_sequence = list(range(length))
_random.shuffle(new_sequence)
else: new_sequence = []
sequence = get_object('sequences-%s' % tag, (lambda *args, **kargs: new_sequence), is_old=is_old)
if length is not None:
sequence = sequence[:length]
return sequence
def get_a_task(task_group_index,
reset=False,
verbose=False,
reset_database=False,
reset_on_run_out=True):
_random = get_object('characters_random', (lambda: random.Random()))
is_old = ((lambda x: reset) if reset else None)
tasks = get_object('recognition-tasks', (lambda *args, **kargs: make_task(
*args, random=_random, verbose=verbose, **kargs)),
is_old=is_old)[task_group_index]
return tasks[sequencer.pop('recognition_characters_%d' % task_group_index,
reset_sequence=reset,
reset_database=reset_database,
reset_on_run_out=reset_on_run_out,
length=len(tasks))]
def get_list(alphabet_id=None, from_path=default_from_path):
raw_input('good')
reg = re.compile(reg_string)
if not use_dict:
def make_dict():
use_dict = {}
for base, dirs, files in os.walk(default_from_path):
print(base)
for file_name in sorted(files):
match = reg.match(file_name)
if match:
name = match.groups()[0]
if name not in use_dict:
use_dict[name] = []
use_dict[name].append(make_path(file_name, desired_base_of_target=default_from_path, actual_base_of_target=base))
return use_dict
use_dict.update(objectstorage.get_object(name, make_dict, timestamp_dir=default_from_path))
if alphabet_id is None:
return use_dict.copy()
if alphabet_id not in use_dict:
return []
if from_path == default_from_path:
return list(use_dict[alphabet_id])
else:
return maplist((lambda i: make_path(i, desired_base_of_target=from_path, actual_base_of_target=default_from_path)), use_dict[alphabet_id])
def make_get_random_task(form, defaults, verbose=False, _random=None):
if _random is None:
_random = random.Random()
alias = form.getfirst("characterSet", default=defaults["characterSet"])
def bad_alias():
raise Exception(
"Character set with the given name does not exist. Run construct-character-set.py to create it."
)
list_of_stuff = get_object("recognition-tasks_character-set_%s" % alias, bad_alias)
if verbose:
print("Getting list of alphabets...")
alphabets_dict = get_accepted_image_list()
trialCount = int(form.getfirst("trialsPerExperiment", default=defaults["trialsPerExperiment"]))
fractionSame = float(form.getfirst("fractionSame", default=defaults["fractionSame"]))
rtn = []
if not isinstance(list_of_stuff[0], (list, tuple)):
alphabets = list_of_stuff
for trial_num in range(trialCount):
same = _random.random() < fractionSame
test = None
while test is None:
alphabet1, alphabet2 = _random.choice(alphabets), _random.choice(alphabets)
if same:
alphabet2 = alphabet1
uid1, uid2 = (
_random.choice(alphabets_dict[alphabet1].keys()),
_random.choice(alphabets_dict[alphabet2].keys()),
)
ch_num1, ch_num2 = (
_random.randrange(len(alphabets_dict[alphabet1][uid1])),
_random.randrange(len(alphabets_dict[alphabet2][uid2])),
)
if same:
ch_num2 = ch_num1
test = [(alphabet1, ch_num1, uid1), (alphabet2, ch_num2, uid2)]
if test[0] == test[1]:
test = None
rtn.append(test)
else:
characters = list_of_stuff
for trial_num in range(trialCount):
same = _random.random() < fractionSame
test = None
while test is None:
(alphabet1, ch_num1), (alphabet2, ch_num2) = _random.choice(characters), _random.choice(characters)
if same:
alphabet2, ch_num2 = alphabet1, ch_num1
uid1, uid2 = (
_random.choice(alphabets_dict[alphabet1].keys()),
_random.choice(alphabets_dict[alphabet2].keys()),
)
test = [(alphabet1, ch_num1, uid1), (alphabet2, ch_num2, uid2)]
if test[0] == test[1]:
test = None
rtn.append(test)
_random.shuffle(rtn)
return rtn
def get_a_task(task_group_index, reset=False, verbose=False, reset_database=False, reset_on_run_out=True):
_random = get_object("characters_random", (lambda: random.Random()))
is_old = (lambda x: reset) if reset else None
tasks = get_object(
"recognition-tasks",
(lambda *args, **kargs: make_task(*args, random=_random, verbose=verbose, **kargs)),
is_old=is_old,
)[task_group_index]
return tasks[
sequencer.pop(
"recognition_characters_%d" % task_group_index,
reset_sequence=reset,
reset_database=reset_database,
reset_on_run_out=reset_on_run_out,
length=len(tasks),
)
]
def make_get_random_task(form, defaults, verbose=False, _random=None):
if _random is None: _random = random.Random()
alias = form.getfirst('characterSet', default=defaults['characterSet'])
def bad_alias():
raise Exception(
"Character set with the given name does not exist. Run construct-character-set.py to create it."
)
list_of_stuff = get_object('recognition-tasks_character-set_%s' % alias,
bad_alias)
if verbose: print('Getting list of alphabets...')
alphabets_dict = get_accepted_image_list()
trialCount = int(
form.getfirst('trialsPerExperiment',
default=defaults['trialsPerExperiment']))
fractionSame = float(
form.getfirst('fractionSame', default=defaults['fractionSame']))
rtn = []
if not isinstance(list_of_stuff[0], (list, tuple)):
alphabets = list_of_stuff
for trial_num in range(trialCount):
same = _random.random() < fractionSame
test = None
while test is None:
alphabet1, alphabet2 = _random.choice(
alphabets), _random.choice(alphabets)
if same: alphabet2 = alphabet1
uid1, uid2 = _random.choice(
alphabets_dict[alphabet1].keys()), _random.choice(
alphabets_dict[alphabet2].keys())
ch_num1, ch_num2 = _random.randrange(
len(alphabets_dict[alphabet1][uid1])), _random.randrange(
len(alphabets_dict[alphabet2][uid2]))
if same: ch_num2 = ch_num1
test = [(alphabet1, ch_num1, uid1), (alphabet2, ch_num2, uid2)]
if test[0] == test[1]: test = None
rtn.append(test)
else:
characters = list_of_stuff
for trial_num in range(trialCount):
same = _random.random() < fractionSame
test = None
while test is None:
(alphabet1, ch_num1), (alphabet2, ch_num2) = _random.choice(
characters), _random.choice(characters)
if same:
alphabet2, ch_num2 = alphabet1, ch_num1
uid1, uid2 = _random.choice(
alphabets_dict[alphabet1].keys()), _random.choice(
alphabets_dict[alphabet2].keys())
test = [(alphabet1, ch_num1, uid1), (alphabet2, ch_num2, uid2)]
if test[0] == test[1]: test = None
rtn.append(test)
_random.shuffle(rtn)
return rtn
def make_task_from_alphabet_set(alias, alphabets_dict=None, **kwargs):
if alphabets_dict is None: alphabets_dict = get_accepted_image_list()
def bad_alias():
raise Exception("Character set with the given name does not exist. Run construct-character-set.py to create it.")
list_of_stuff = get_object('recognition-tasks_character-set_%s' % alias, bad_alias)
if not isinstance(list_of_stuff[0], (list, tuple)):
return make_task_from_alphabets_list(list_of_stuff, alphabets_dict=alphabets_dict, **kwargs)
else:
return make_task_from_groups_list(list_of_stuff, alphabets_dict=alphabets_dict, **kwargs)
def make_task_from_alphabet_set(alphabet_set, alphabets_dict=None, **kwargs):
if alphabets_dict is None: alphabets_dict = get_accepted_image_list()
def bad_alias():
raise Exception("Character set with the given name does not exist. Run construct-character-set.py to create it.")
list_of_stuff = get_object('recognition-tasks_character-set_%s' % alphabet_set, bad_alias)
if not isinstance(list_of_stuff[0], (list, tuple)):
return make_task_from_alphabet_list(list_of_stuff, alphabets_dict=alphabets_dict, **kwargs)
else: # is string
return make_task_from_characters_list(list_of_stuff, alphabets_dict=alphabets_dict, **kwargs)
def get_list(alphabet_id=None, id_=None, from_path=default_from_path):
if not use_dict:
def make_dict():
use_dict = {}
for base, dirs, files in os.walk(base_dir):
print(base)
for file_name in sorted(files):
match = reg.match(file_name)
if match:
name, number, cur_id = match.groups()
name = name
if name not in use_dict: use_dict[name] = {}
if cur_id not in use_dict[name]: use_dict[name][cur_id] = []
use_dict[name][cur_id].append(make_path(file_name, desired_base_of_target=default_from_path, actual_base_of_target=base))
return use_dict
use_dict.update(objectstorage.get_object(name, make_dict, timestamp_dir=default_from_path))
if default_from_path == from_path:
def fix_path(path): return path
else:
def fix_path(path): return make_path(path, desired_base_of_target=from_path, actual_base_of_target=default_from_path)
if alphabet_id is None:
rtn = {}
if id_ is None:
for alphabet_id in use_dict:
rtn[alphabet_id] = {}
for id_ in use_dict[alphabet_id]:
rtn[alphabet_id][id_] = maplist(fix_path, use_dict[alphabet_id][id_])
else:
rtn = {}
for alphabet_id in use_dict:
if id_ in use_dict[alphabet_id]:
rtn[alphabet_id] = maplist(fix_path, use_dict[alphabet_id][id_])
return rtn
else:
if id_ is None:
if alphabet_id not in use_dict: return {}
rtn = {}
for cur_id in use_dict[alphabet_id]:
rtn[cur_id] = maplist(fix_path, use_dict[alphabet_id][cur_id])
return rtn
else:
if alphabet_id not in use_dict: return []
return maplist((lambda i: relpath(i, from_path)), use_dict[alphabet_id][id_])
def get_hashed_images_dict():
return objectstorage.get_object('get_hashed_images_dict', _make_hashed_images_dict, timestamp_dir=ACCEPTED_IMAGES_PATH)
#!/usr/bin/python
from alphabetspaths import *
import objectstorage
from google import search
import re
__all__ = ['get_popularity']
_polularity_dict = objectstorage.get_object(
'alphabetspopularity-popularity_dict', (lambda: {}))
_popularity_override = {}
_polularity_dict.update(_popularity_override)
temp = ''
_reg = re.compile(r'<div id="resultStats">\s*About ([0-9]+) results\s*<nobr>')
def get_popularity(
alphabet,
reset=False,
transform=(lambda s: re.sub(r'(.*?) \((.*?)\)', r'"\1" OR "\2"',
s.lower().replace('-', '')))):
possibilities = (alphabet, get_alphabet_name(alphabet))
if not reset:
for poss in possibilities:
if poss.lower() in _polularity_dict:
return _polularity_dict[poss.lower()]
else:
for poss in possibilities:
if poss.lower() in _popularity_override:
def construct_character_set(form, args, reset=False, verbose=False, help=False):
if get_boolean_value(form, 'help', default=help):
print('Content-type: text/html\n')
print("""<p>You may any of the following permissible url parameters:</p>
<ul>
<li>
<b>characters</b> - A list of characters that must be used, in the form [<em>alphabet</em>, <em>number</em>], where <em>number</em> is a
one-based index. For example, <tt>?characters=[["latin", 1], ["latin", 2], ["greek", 10]]</tt>.
<span style="color:red"><strong>Note that the quotes (") are mandatory.</strong></span>
</li>
<li>
<b>alphabets</b> - A list of alphabets from which to draw the characters. For example,
<tt>?constructCharacterSet&alphabets=["latin", "greek", "hebrew"]</tt>. Does not work well with <b>nonAlphabets</b>.
<span style="color:red"><strong>Note that the quotes (") are mandatory.</strong></span>
</li>
<li>
<b>drawers</b> - A list of alphabet/drawer pairs that must be used, in the form [<em>alphabet</em>, <em>id</em>].
For example, <tt>?drawers=[["latin", "a1j8s7giuyto4a"], ["latin", "a1kj5bqqwzijjt"], ["greek", "a2pfktghhg1ofp"]]</tt>.
<span style="color:red"><strong>Note that the quotes (") are mandatory.</strong></span>
</li>
<li>
<b>nonAlphabets</b> - A list of alphabets from which characters may not be drawn. For example,
<tt>?constructCharacterSet&nonAlphabets=["latin", "greek", "hebrew"]</tt>. Does not work well with <b>alphabets</b>.
<span style="color:red"><strong>Note that the quotes (") are mandatory.</strong></span>
</li>
<li><b>totalCharacters</b> - the total number of characters to draw. </li>
<li><b>alias</b> - the name of the structure. If you do not choose an alias, you will be given a hash code. </li>
<li><b>overwrite</b> - whether or not an already existent character set should be overwritten. Either <tt>true</tt> or <tt>false</tt>.</li>
<li><b>get</b> - If you want to see the current contents of a character set, use this parameter. For example, <tt>?get=small</tt>.</li>
</ul>""")
return False
try:
get_alias = form.getfirst('get', args.get)
if get_alias:
def do_error():
raise ValueError('No structure with alias "%s" exists.' % get_alias)
return get_object(CHARACTER_SET_NAME_FORMAT % get_alias, do_error, is_old=(lambda x: False))
drawers = get_list_of_values(form, 'drawers', args.drawers)
characters = get_list_of_values(form, 'characters', args.characters)
alphabets = get_list_of_values(form, 'alphabets', args.alphabets)
non_alphabets = get_list_of_values(form, 'nonAlphabets', args.non_alphabets)
totalCharacters = int(form.getfirst('totalCharacters', args.total_characters))
alias = form.getfirst('alias', args.alias)
overwrite = get_boolean_value(form, 'overwrite', default=((not args.no_overwrite) and (args.overwrite or (alias is None))))
if characters:
rtn = [(alphabet, int(ch_num) - 1) for alphabet, ch_num in characters]
elif alphabets:
rtn = alphabets
elif drawers:
rtn = [(alphabet, '*', uid) for alphabet, uid in drawers]
elif non_alphabets:
rtn = [alphabet for alphabet in get_accepted_image_list()
if alphabet not in nonAlphabets and alphabet.lower() not in nonAlphabets]
else:
raise Exception("Insufficient parameters for decision")
rtn.sort()
if not alias:
alias = str(hash(rtn))
except Exception:
construct_character_set(form, args, help=True)
raise
if rtn != get_object(CHARACTER_SET_NAME_FORMAT % alias, (lambda: rtn), is_old=(lambda x: overwrite)):
raise Exception('Structure with given alias (%s) already exists. Pick a new alias.' % alias)
return {'alias':alias}
def construct_character_set(form,
args,
reset=False,
verbose=False,
help=False):
if get_boolean_value(form, 'help', default=help):
print('Content-type: text/html\n')
print(
"""<p>You may any of the following permissible url parameters:</p>
<ul>
<li>
<b>characters</b> - A list of characters that must be used, in the form [<em>alphabet</em>, <em>number</em>], where <em>number</em> is a
one-based index. For example, <tt>?characters=[["latin", 1], ["latin", 2], ["greek", 10]]</tt>.
<span style="color:red"><strong>Note that the quotes (") are mandatory.</strong></span>
</li>
<li>
<b>alphabets</b> - A list of alphabets from which to draw the characters. For example,
<tt>?constructCharacterSet&alphabets=["latin", "greek", "hebrew"]</tt>. Does not work well with <b>nonAlphabets</b>.
<span style="color:red"><strong>Note that the quotes (") are mandatory.</strong></span>
</li>
<li>
<b>drawers</b> - A list of alphabet/drawer pairs that must be used, in the form [<em>alphabet</em>, <em>id</em>].
For example, <tt>?drawers=[["latin", "a1j8s7giuyto4a"], ["latin", "a1kj5bqqwzijjt"], ["greek", "a2pfktghhg1ofp"]]</tt>.
<span style="color:red"><strong>Note that the quotes (") are mandatory.</strong></span>
</li>
<li>
<b>nonAlphabets</b> - A list of alphabets from which characters may not be drawn. For example,
<tt>?constructCharacterSet&nonAlphabets=["latin", "greek", "hebrew"]</tt>. Does not work well with <b>alphabets</b>.
<span style="color:red"><strong>Note that the quotes (") are mandatory.</strong></span>
</li>
<li><b>totalCharacters</b> - the total number of characters to draw. </li>
<li><b>alias</b> - the name of the structure. If you do not choose an alias, you will be given a hash code. </li>
<li><b>overwrite</b> - whether or not an already existent character set should be overwritten. Either <tt>true</tt> or <tt>false</tt>.</li>
<li><b>get</b> - If you want to see the current contents of a character set, use this parameter. For example, <tt>?get=small</tt>.</li>
</ul>""")
return False
try:
get_alias = form.getfirst('get', args.get)
if get_alias:
def do_error():
raise ValueError('No structure with alias "%s" exists.' %
get_alias)
return get_object(CHARACTER_SET_NAME_FORMAT % get_alias,
do_error,
is_old=(lambda x: False))
drawers = get_list_of_values(form, 'drawers', args.drawers)
characters = get_list_of_values(form, 'characters', args.characters)
alphabets = get_list_of_values(form, 'alphabets', args.alphabets)
non_alphabets = get_list_of_values(form, 'nonAlphabets',
args.non_alphabets)
totalCharacters = int(
form.getfirst('totalCharacters', args.total_characters))
alias = form.getfirst('alias', args.alias)
overwrite = get_boolean_value(form,
'overwrite',
default=((not args.no_overwrite)
and (args.overwrite or
(alias is None))))
if characters:
rtn = [(alphabet, int(ch_num) - 1)
for alphabet, ch_num in characters]
elif alphabets:
rtn = alphabets
elif drawers:
rtn = [(alphabet, '*', uid) for alphabet, uid in drawers]
elif non_alphabets:
rtn = [
alphabet for alphabet in get_accepted_image_list()
if alphabet not in nonAlphabets
and alphabet.lower() not in nonAlphabets
]
else:
raise Exception("Insufficient parameters for decision")
rtn.sort()
if not alias:
alias = str(hash(rtn))
except Exception:
construct_character_set(form, args, help=True)
raise
if rtn != get_object(CHARACTER_SET_NAME_FORMAT % alias, (lambda: rtn),
is_old=(lambda x: overwrite)):
raise Exception(
'Structure with given alias (%s) already exists. Pick a new alias.'
% alias)
return {'alias': alias}
#!/usr/bin/python
from alphabetspaths import *
import objectstorage
from google import search
import re
__all__ = ['get_popularity']
_polularity_dict = objectstorage.get_object('alphabetspopularity-popularity_dict', (lambda: {}))
_popularity_override = {}
_polularity_dict.update(_popularity_override)
temp = ''
_reg = re.compile(r'<div id="resultStats">\s*About ([0-9]+) results\s*<nobr>')
def get_popularity(alphabet, reset=False, transform=(lambda s:re.sub(r'(.*?) \((.*?)\)', r'"\1" OR "\2"', s.lower().replace('-','')))):
possibilities = (alphabet, get_alphabet_name(alphabet))
if not reset:
for poss in possibilities:
if poss.lower() in _polularity_dict:
return _polularity_dict[poss.lower()]
else:
for poss in possibilities:
if poss.lower() in _popularity_override:
return _popularity_override[poss.lower()]
if get_alphabet_name(alphabet): alphabet_name = get_alphabet_name(alphabet)
else: alphabet_name = alphabet
_polularity_dict[alphabet_name.lower()] = get_number_of_results(alphabet_name.replace('-', ''))
if transform(alphabet_name) not in (alphabet_name.lower(), alphabet_name, alphabet_name.replace('-', ''), alpahbet_name.lower().replace('-', '')):
_polularity_dict[alphabet_name.lower()] = max((_polularity_dict[alphabet_name.lower()], get_number_of_results(transform(alphabet_name))))
objectstorage.save_object('alphabetspopularity-popularity_dict', _polularity_dict)
