def get_random_sequence(tag, length=None, reset=False): _random = get_object('sequencer-random', (lambda: random.Random())) is_old = ((lambda x:reset) if reset else None) if length is not None: new_sequence = list(range(length)) _random.shuffle(new_sequence) else: new_sequence = [] sequence = get_object('sequences-%s' % tag, (lambda *args, **kargs: new_sequence), is_old=is_old) if length is not None: sequence = sequence[:length] return sequence
def get_a_task(task_group_index, reset=False, verbose=False, reset_database=False, reset_on_run_out=True): _random = get_object('characters_random', (lambda: random.Random())) is_old = ((lambda x: reset) if reset else None) tasks = get_object('recognition-tasks', (lambda *args, **kargs: make_task( *args, random=_random, verbose=verbose, **kargs)), is_old=is_old)[task_group_index] return tasks[sequencer.pop('recognition_characters_%d' % task_group_index, reset_sequence=reset, reset_database=reset_database, reset_on_run_out=reset_on_run_out, length=len(tasks))]
def get_list(alphabet_id=None, from_path=default_from_path): raw_input('good') reg = re.compile(reg_string) if not use_dict: def make_dict(): use_dict = {} for base, dirs, files in os.walk(default_from_path): print(base) for file_name in sorted(files): match = reg.match(file_name) if match: name = match.groups()[0] if name not in use_dict: use_dict[name] = [] use_dict[name].append(make_path(file_name, desired_base_of_target=default_from_path, actual_base_of_target=base)) return use_dict use_dict.update(objectstorage.get_object(name, make_dict, timestamp_dir=default_from_path)) if alphabet_id is None: return use_dict.copy() if alphabet_id not in use_dict: return [] if from_path == default_from_path: return list(use_dict[alphabet_id]) else: return maplist((lambda i: make_path(i, desired_base_of_target=from_path, actual_base_of_target=default_from_path)), use_dict[alphabet_id])
def make_get_random_task(form, defaults, verbose=False, _random=None): if _random is None: _random = random.Random() alias = form.getfirst("characterSet", default=defaults["characterSet"]) def bad_alias(): raise Exception( "Character set with the given name does not exist. Run construct-character-set.py to create it." ) list_of_stuff = get_object("recognition-tasks_character-set_%s" % alias, bad_alias) if verbose: print("Getting list of alphabets...") alphabets_dict = get_accepted_image_list() trialCount = int(form.getfirst("trialsPerExperiment", default=defaults["trialsPerExperiment"])) fractionSame = float(form.getfirst("fractionSame", default=defaults["fractionSame"])) rtn = [] if not isinstance(list_of_stuff[0], (list, tuple)): alphabets = list_of_stuff for trial_num in range(trialCount): same = _random.random() < fractionSame test = None while test is None: alphabet1, alphabet2 = _random.choice(alphabets), _random.choice(alphabets) if same: alphabet2 = alphabet1 uid1, uid2 = ( _random.choice(alphabets_dict[alphabet1].keys()), _random.choice(alphabets_dict[alphabet2].keys()), ) ch_num1, ch_num2 = ( _random.randrange(len(alphabets_dict[alphabet1][uid1])), _random.randrange(len(alphabets_dict[alphabet2][uid2])), ) if same: ch_num2 = ch_num1 test = [(alphabet1, ch_num1, uid1), (alphabet2, ch_num2, uid2)] if test[0] == test[1]: test = None rtn.append(test) else: characters = list_of_stuff for trial_num in range(trialCount): same = _random.random() < fractionSame test = None while test is None: (alphabet1, ch_num1), (alphabet2, ch_num2) = _random.choice(characters), _random.choice(characters) if same: alphabet2, ch_num2 = alphabet1, ch_num1 uid1, uid2 = ( _random.choice(alphabets_dict[alphabet1].keys()), _random.choice(alphabets_dict[alphabet2].keys()), ) test = [(alphabet1, ch_num1, uid1), (alphabet2, ch_num2, uid2)] if test[0] == test[1]: test = None rtn.append(test) _random.shuffle(rtn) return rtn
def get_a_task(task_group_index, reset=False, verbose=False, reset_database=False, reset_on_run_out=True): _random = get_object("characters_random", (lambda: random.Random())) is_old = (lambda x: reset) if reset else None tasks = get_object( "recognition-tasks", (lambda *args, **kargs: make_task(*args, random=_random, verbose=verbose, **kargs)), is_old=is_old, )[task_group_index] return tasks[ sequencer.pop( "recognition_characters_%d" % task_group_index, reset_sequence=reset, reset_database=reset_database, reset_on_run_out=reset_on_run_out, length=len(tasks), ) ]
def make_get_random_task(form, defaults, verbose=False, _random=None): if _random is None: _random = random.Random() alias = form.getfirst('characterSet', default=defaults['characterSet']) def bad_alias(): raise Exception( "Character set with the given name does not exist. Run construct-character-set.py to create it." ) list_of_stuff = get_object('recognition-tasks_character-set_%s' % alias, bad_alias) if verbose: print('Getting list of alphabets...') alphabets_dict = get_accepted_image_list() trialCount = int( form.getfirst('trialsPerExperiment', default=defaults['trialsPerExperiment'])) fractionSame = float( form.getfirst('fractionSame', default=defaults['fractionSame'])) rtn = [] if not isinstance(list_of_stuff[0], (list, tuple)): alphabets = list_of_stuff for trial_num in range(trialCount): same = _random.random() < fractionSame test = None while test is None: alphabet1, alphabet2 = _random.choice( alphabets), _random.choice(alphabets) if same: alphabet2 = alphabet1 uid1, uid2 = _random.choice( alphabets_dict[alphabet1].keys()), _random.choice( alphabets_dict[alphabet2].keys()) ch_num1, ch_num2 = _random.randrange( len(alphabets_dict[alphabet1][uid1])), _random.randrange( len(alphabets_dict[alphabet2][uid2])) if same: ch_num2 = ch_num1 test = [(alphabet1, ch_num1, uid1), (alphabet2, ch_num2, uid2)] if test[0] == test[1]: test = None rtn.append(test) else: characters = list_of_stuff for trial_num in range(trialCount): same = _random.random() < fractionSame test = None while test is None: (alphabet1, ch_num1), (alphabet2, ch_num2) = _random.choice( characters), _random.choice(characters) if same: alphabet2, ch_num2 = alphabet1, ch_num1 uid1, uid2 = _random.choice( alphabets_dict[alphabet1].keys()), _random.choice( alphabets_dict[alphabet2].keys()) test = [(alphabet1, ch_num1, uid1), (alphabet2, ch_num2, uid2)] if test[0] == test[1]: test = None rtn.append(test) _random.shuffle(rtn) return rtn
def make_task_from_alphabet_set(alias, alphabets_dict=None, **kwargs): if alphabets_dict is None: alphabets_dict = get_accepted_image_list() def bad_alias(): raise Exception("Character set with the given name does not exist. Run construct-character-set.py to create it.") list_of_stuff = get_object('recognition-tasks_character-set_%s' % alias, bad_alias) if not isinstance(list_of_stuff[0], (list, tuple)): return make_task_from_alphabets_list(list_of_stuff, alphabets_dict=alphabets_dict, **kwargs) else: return make_task_from_groups_list(list_of_stuff, alphabets_dict=alphabets_dict, **kwargs)
def make_task_from_alphabet_set(alphabet_set, alphabets_dict=None, **kwargs): if alphabets_dict is None: alphabets_dict = get_accepted_image_list() def bad_alias(): raise Exception("Character set with the given name does not exist. Run construct-character-set.py to create it.") list_of_stuff = get_object('recognition-tasks_character-set_%s' % alphabet_set, bad_alias) if not isinstance(list_of_stuff[0], (list, tuple)): return make_task_from_alphabet_list(list_of_stuff, alphabets_dict=alphabets_dict, **kwargs) else: # is string return make_task_from_characters_list(list_of_stuff, alphabets_dict=alphabets_dict, **kwargs)
def get_list(alphabet_id=None, id_=None, from_path=default_from_path): if not use_dict: def make_dict(): use_dict = {} for base, dirs, files in os.walk(base_dir): print(base) for file_name in sorted(files): match = reg.match(file_name) if match: name, number, cur_id = match.groups() name = name if name not in use_dict: use_dict[name] = {} if cur_id not in use_dict[name]: use_dict[name][cur_id] = [] use_dict[name][cur_id].append(make_path(file_name, desired_base_of_target=default_from_path, actual_base_of_target=base)) return use_dict use_dict.update(objectstorage.get_object(name, make_dict, timestamp_dir=default_from_path)) if default_from_path == from_path: def fix_path(path): return path else: def fix_path(path): return make_path(path, desired_base_of_target=from_path, actual_base_of_target=default_from_path) if alphabet_id is None: rtn = {} if id_ is None: for alphabet_id in use_dict: rtn[alphabet_id] = {} for id_ in use_dict[alphabet_id]: rtn[alphabet_id][id_] = maplist(fix_path, use_dict[alphabet_id][id_]) else: rtn = {} for alphabet_id in use_dict: if id_ in use_dict[alphabet_id]: rtn[alphabet_id] = maplist(fix_path, use_dict[alphabet_id][id_]) return rtn else: if id_ is None: if alphabet_id not in use_dict: return {} rtn = {} for cur_id in use_dict[alphabet_id]: rtn[cur_id] = maplist(fix_path, use_dict[alphabet_id][cur_id]) return rtn else: if alphabet_id not in use_dict: return [] return maplist((lambda i: relpath(i, from_path)), use_dict[alphabet_id][id_])
def get_hashed_images_dict(): return objectstorage.get_object('get_hashed_images_dict', _make_hashed_images_dict, timestamp_dir=ACCEPTED_IMAGES_PATH)
#!/usr/bin/python from alphabetspaths import * import objectstorage from google import search import re __all__ = ['get_popularity'] _polularity_dict = objectstorage.get_object( 'alphabetspopularity-popularity_dict', (lambda: {})) _popularity_override = {} _polularity_dict.update(_popularity_override) temp = '' _reg = re.compile(r'<div id="resultStats">\s*About ([0-9]+) results\s*<nobr>') def get_popularity( alphabet, reset=False, transform=(lambda s: re.sub(r'(.*?) \((.*?)\)', r'"\1" OR "\2"', s.lower().replace('-', '')))): possibilities = (alphabet, get_alphabet_name(alphabet)) if not reset: for poss in possibilities: if poss.lower() in _polularity_dict: return _polularity_dict[poss.lower()] else: for poss in possibilities: if poss.lower() in _popularity_override:
def construct_character_set(form, args, reset=False, verbose=False, help=False): if get_boolean_value(form, 'help', default=help): print('Content-type: text/html\n') print("""<p>You may any of the following permissible url parameters:</p> <ul> <li> <b>characters</b> - A list of characters that must be used, in the form [<em>alphabet</em>, <em>number</em>], where <em>number</em> is a one-based index. For example, <tt>?characters=[["latin", 1], ["latin", 2], ["greek", 10]]</tt>. <span style="color:red"><strong>Note that the quotes (") are mandatory.</strong></span> </li> <li> <b>alphabets</b> - A list of alphabets from which to draw the characters. For example, <tt>?constructCharacterSet&alphabets=["latin", "greek", "hebrew"]</tt>. Does not work well with <b>nonAlphabets</b>. <span style="color:red"><strong>Note that the quotes (") are mandatory.</strong></span> </li> <li> <b>drawers</b> - A list of alphabet/drawer pairs that must be used, in the form [<em>alphabet</em>, <em>id</em>]. For example, <tt>?drawers=[["latin", "a1j8s7giuyto4a"], ["latin", "a1kj5bqqwzijjt"], ["greek", "a2pfktghhg1ofp"]]</tt>. <span style="color:red"><strong>Note that the quotes (") are mandatory.</strong></span> </li> <li> <b>nonAlphabets</b> - A list of alphabets from which characters may not be drawn. For example, <tt>?constructCharacterSet&nonAlphabets=["latin", "greek", "hebrew"]</tt>. Does not work well with <b>alphabets</b>. <span style="color:red"><strong>Note that the quotes (") are mandatory.</strong></span> </li> <li><b>totalCharacters</b> - the total number of characters to draw. </li> <li><b>alias</b> - the name of the structure. If you do not choose an alias, you will be given a hash code. </li> <li><b>overwrite</b> - whether or not an already existent character set should be overwritten. Either <tt>true</tt> or <tt>false</tt>.</li> <li><b>get</b> - If you want to see the current contents of a character set, use this parameter. For example, <tt>?get=small</tt>.</li> </ul>""") return False try: get_alias = form.getfirst('get', args.get) if get_alias: def do_error(): raise ValueError('No structure with alias "%s" exists.' % get_alias) return get_object(CHARACTER_SET_NAME_FORMAT % get_alias, do_error, is_old=(lambda x: False)) drawers = get_list_of_values(form, 'drawers', args.drawers) characters = get_list_of_values(form, 'characters', args.characters) alphabets = get_list_of_values(form, 'alphabets', args.alphabets) non_alphabets = get_list_of_values(form, 'nonAlphabets', args.non_alphabets) totalCharacters = int(form.getfirst('totalCharacters', args.total_characters)) alias = form.getfirst('alias', args.alias) overwrite = get_boolean_value(form, 'overwrite', default=((not args.no_overwrite) and (args.overwrite or (alias is None)))) if characters: rtn = [(alphabet, int(ch_num) - 1) for alphabet, ch_num in characters] elif alphabets: rtn = alphabets elif drawers: rtn = [(alphabet, '*', uid) for alphabet, uid in drawers] elif non_alphabets: rtn = [alphabet for alphabet in get_accepted_image_list() if alphabet not in nonAlphabets and alphabet.lower() not in nonAlphabets] else: raise Exception("Insufficient parameters for decision") rtn.sort() if not alias: alias = str(hash(rtn)) except Exception: construct_character_set(form, args, help=True) raise if rtn != get_object(CHARACTER_SET_NAME_FORMAT % alias, (lambda: rtn), is_old=(lambda x: overwrite)): raise Exception('Structure with given alias (%s) already exists. Pick a new alias.' % alias) return {'alias':alias}
def construct_character_set(form, args, reset=False, verbose=False, help=False): if get_boolean_value(form, 'help', default=help): print('Content-type: text/html\n') print( """<p>You may any of the following permissible url parameters:</p> <ul> <li> <b>characters</b> - A list of characters that must be used, in the form [<em>alphabet</em>, <em>number</em>], where <em>number</em> is a one-based index. For example, <tt>?characters=[["latin", 1], ["latin", 2], ["greek", 10]]</tt>. <span style="color:red"><strong>Note that the quotes (") are mandatory.</strong></span> </li> <li> <b>alphabets</b> - A list of alphabets from which to draw the characters. For example, <tt>?constructCharacterSet&alphabets=["latin", "greek", "hebrew"]</tt>. Does not work well with <b>nonAlphabets</b>. <span style="color:red"><strong>Note that the quotes (") are mandatory.</strong></span> </li> <li> <b>drawers</b> - A list of alphabet/drawer pairs that must be used, in the form [<em>alphabet</em>, <em>id</em>]. For example, <tt>?drawers=[["latin", "a1j8s7giuyto4a"], ["latin", "a1kj5bqqwzijjt"], ["greek", "a2pfktghhg1ofp"]]</tt>. <span style="color:red"><strong>Note that the quotes (") are mandatory.</strong></span> </li> <li> <b>nonAlphabets</b> - A list of alphabets from which characters may not be drawn. For example, <tt>?constructCharacterSet&nonAlphabets=["latin", "greek", "hebrew"]</tt>. Does not work well with <b>alphabets</b>. <span style="color:red"><strong>Note that the quotes (") are mandatory.</strong></span> </li> <li><b>totalCharacters</b> - the total number of characters to draw. </li> <li><b>alias</b> - the name of the structure. If you do not choose an alias, you will be given a hash code. </li> <li><b>overwrite</b> - whether or not an already existent character set should be overwritten. Either <tt>true</tt> or <tt>false</tt>.</li> <li><b>get</b> - If you want to see the current contents of a character set, use this parameter. For example, <tt>?get=small</tt>.</li> </ul>""") return False try: get_alias = form.getfirst('get', args.get) if get_alias: def do_error(): raise ValueError('No structure with alias "%s" exists.' % get_alias) return get_object(CHARACTER_SET_NAME_FORMAT % get_alias, do_error, is_old=(lambda x: False)) drawers = get_list_of_values(form, 'drawers', args.drawers) characters = get_list_of_values(form, 'characters', args.characters) alphabets = get_list_of_values(form, 'alphabets', args.alphabets) non_alphabets = get_list_of_values(form, 'nonAlphabets', args.non_alphabets) totalCharacters = int( form.getfirst('totalCharacters', args.total_characters)) alias = form.getfirst('alias', args.alias) overwrite = get_boolean_value(form, 'overwrite', default=((not args.no_overwrite) and (args.overwrite or (alias is None)))) if characters: rtn = [(alphabet, int(ch_num) - 1) for alphabet, ch_num in characters] elif alphabets: rtn = alphabets elif drawers: rtn = [(alphabet, '*', uid) for alphabet, uid in drawers] elif non_alphabets: rtn = [ alphabet for alphabet in get_accepted_image_list() if alphabet not in nonAlphabets and alphabet.lower() not in nonAlphabets ] else: raise Exception("Insufficient parameters for decision") rtn.sort() if not alias: alias = str(hash(rtn)) except Exception: construct_character_set(form, args, help=True) raise if rtn != get_object(CHARACTER_SET_NAME_FORMAT % alias, (lambda: rtn), is_old=(lambda x: overwrite)): raise Exception( 'Structure with given alias (%s) already exists. Pick a new alias.' % alias) return {'alias': alias}
#!/usr/bin/python from alphabetspaths import * import objectstorage from google import search import re __all__ = ['get_popularity'] _polularity_dict = objectstorage.get_object('alphabetspopularity-popularity_dict', (lambda: {})) _popularity_override = {} _polularity_dict.update(_popularity_override) temp = '' _reg = re.compile(r'<div id="resultStats">\s*About ([0-9]+) results\s*<nobr>') def get_popularity(alphabet, reset=False, transform=(lambda s:re.sub(r'(.*?) \((.*?)\)', r'"\1" OR "\2"', s.lower().replace('-','')))): possibilities = (alphabet, get_alphabet_name(alphabet)) if not reset: for poss in possibilities: if poss.lower() in _polularity_dict: return _polularity_dict[poss.lower()] else: for poss in possibilities: if poss.lower() in _popularity_override: return _popularity_override[poss.lower()] if get_alphabet_name(alphabet): alphabet_name = get_alphabet_name(alphabet) else: alphabet_name = alphabet _polularity_dict[alphabet_name.lower()] = get_number_of_results(alphabet_name.replace('-', '')) if transform(alphabet_name) not in (alphabet_name.lower(), alphabet_name, alphabet_name.replace('-', ''), alpahbet_name.lower().replace('-', '')): _polularity_dict[alphabet_name.lower()] = max((_polularity_dict[alphabet_name.lower()], get_number_of_results(transform(alphabet_name)))) objectstorage.save_object('alphabetspopularity-popularity_dict', _polularity_dict)