def get_takes_last_week(self): """Get the users' nominal takes last week. Used in throttling. """ assert self.kind == 'group' takes = OrderedDict((t.member, t.amount) for t in self.db.all( """ SELECT DISTINCT ON (member) member, amount FROM takes WHERE team=%s AND mtime < ( SELECT ts_start FROM paydays WHERE ts_end > ts_start ORDER BY ts_start DESC LIMIT 1 ) ORDER BY member, mtime DESC """, (self.id, )) if t.amount) takes.nonzero = any(amount != 0 for amount in takes.values()) takes.sum = MoneyBasket(amount for amount in takes.values() if amount > 0) takes.initial_leftover = self.receiving - takes.sum.fuzzy_sum( self.main_currency) return takes
class FractionTaxaBarStack(Graph): """Comparing all fractions across all pools in a barstack""" short_name = 'fraction_taxa_barstack' def plot(self): self.frame = OrderedDict((('%s - %s' % (p,f), getattr(p.fractions, f).rdp.phyla) for f in ('low', 'med', 'big') for p in self.parent.pools)) self.frame = pandas.DataFrame(self.frame) self.frame = self.frame.fillna(0) self.frame = self.frame.transpose() self.frame = self.frame.apply(lambda x: 100*x/x.sum(), axis=1) # Sort the table by sum # sums = self.frame.sum() sums.sort(ascending=False) self.frame = self.frame.reindex_axis(sums.keys(), axis=1) # Plot # fig = pyplot.figure() axes = self.frame.plot(kind='bar', stacked=True, color=cool_colors) fig = pyplot.gcf() # Other # axes.set_title('Species relative abundances per fraction per pool') axes.set_ylabel('Relative abundances in percent') axes.xaxis.grid(False) axes.yaxis.grid(False) axes.set_ylim([0,100]) # Put a legend below current axis axes.legend(loc='upper center', bbox_to_anchor=(0.5, -0.20), fancybox=True, shadow=True, ncol=5) # Save it # self.save_plot(fig, axes, width=24.0, height=14.0, bottom=0.30, top=0.97, left=0.04, right=0.98) self.frame.to_csv(self.csv_path) pyplot.close(fig)
class FractionTaxaBarStack(Graph): """Comparing all fractions across all pools in a barstack""" short_name = 'fraction_taxa_barstack' def plot(self): self.frame = OrderedDict( (('%s - %s' % (p, f), getattr(p.fractions, f).rdp.phyla) for f in ('low', 'med', 'big') for p in self.parent.pools)) self.frame = pandas.DataFrame(self.frame) self.frame = self.frame.fillna(0) self.frame = self.frame.transpose() self.frame = self.frame.apply(lambda x: 100 * x / x.sum(), axis=1) # Sort the table by sum # sums = self.frame.sum() sums.sort(ascending=False) self.frame = self.frame.reindex_axis(sums.keys(), axis=1) # Plot # fig = pyplot.figure() axes = self.frame.plot(kind='bar', stacked=True, color=cool_colors) fig = pyplot.gcf() # Other # axes.set_title('Species relative abundances per fraction per pool') axes.set_ylabel('Relative abundances in percent') axes.xaxis.grid(False) axes.yaxis.grid(False) axes.set_ylim([0, 100]) # Put a legend below current axis axes.legend(loc='upper center', bbox_to_anchor=(0.5, -0.20), fancybox=True, shadow=True, ncol=5) # Save it # self.save_plot(fig, axes, width=24.0, height=14.0, bottom=0.30, top=0.97, left=0.04, right=0.98) self.frame.to_csv(self.csv_path) pyplot.close(fig)
def get_takes_last_week(self): """Get the users' nominal takes last week. Used in throttling. """ assert self.kind == 'group' takes = OrderedDict((t.member, t.amount) for t in self.db.all(""" SELECT DISTINCT ON (member) member, amount, mtime FROM takes WHERE team=%s AND mtime < ( SELECT ts_start FROM paydays WHERE ts_end > ts_start ORDER BY ts_start DESC LIMIT 1 ) ORDER BY member, mtime DESC """, (self.id,)) if t.amount) takes.sum = MoneyBasket(takes.values()) takes.initial_leftover = self.get_exact_receiving() - takes.sum return takes
def get_takes_last_week(self): """Get the users' nominal takes last week. Used in throttling. """ assert self.kind == 'group' takes = OrderedDict((t.member, t.amount) for t in self.db.all(""" SELECT DISTINCT ON (member) member, amount FROM takes WHERE team=%s AND mtime < ( SELECT ts_start FROM paydays WHERE ts_end > ts_start ORDER BY ts_start DESC LIMIT 1 ) ORDER BY member, mtime DESC """, (self.id,)) if t.amount is not None) takes.nonzero = any(amount != 0 for amount in takes.values()) takes.sum = MoneyBasket(amount for amount in takes.values() if amount > 0) takes.initial_leftover = self.receiving - takes.sum.fuzzy_sum(self.main_currency) return takes
def pmultiquery(corpus, search, show = 'words', query = 'any', sort_by = 'total', quicksave = False, multiprocess = 'default', function_filter = False, just_speakers = False, root = False, note = False, print_info = True, **kwargs): """Parallel process multiple queries or corpora. This function is used by interrogator() if: a) path is a list of paths b) query is a dict of named queries c) just speakers == 'each', or a list of speakers with len(list) > 1 This function needs joblib 0.8.4 or above in order to run properly. There's no reason to call it yourself.""" import collections import os import pandas as pd import collections from collections import namedtuple from time import strftime, localtime import corpkit from interrogator import interrogator from editor import editor from other import save from interrogation import Interrogation try: from joblib import Parallel, delayed except: pass #raise ValueError('joblib, the module used for multiprocessing, cannot be found. ' \ # 'Install with:\n\n pip install joblib') import multiprocessing def best_num_parallel(num_cores, num_queries): import corpkit """decide how many parallel processes to run the idea, more or less, is to balance the load when possible""" if num_queries <= num_cores: return num_queries if num_queries > num_cores: if (num_queries / num_cores) == num_cores: return int(num_cores) if num_queries % num_cores == 0: try: return max([int(num_queries / n) for n in range(2, num_cores) if int(num_queries / n) <= num_cores]) except ValueError: return num_cores else: import math if (float(math.sqrt(num_queries))).is_integer(): square_root = math.sqrt(num_queries) if square_root <= num_queries / num_cores: return int(square_root) return num_cores num_cores = multiprocessing.cpu_count() # what is our iterable? ... multiple_option = False multiple_queries = False multiple_speakers = False multiple_corpora = False multiple_search = False mult_corp_are_subs = False denom = 1 if hasattr(corpus, '__iter__'): multiple_corpora = True num_cores = best_num_parallel(num_cores, len(corpus)) denom = len(corpus) if all(c.__class__ == corpkit.corpus.Subcorpus for c in corpus): mult_corp_are_subs = True elif (type(query) == list or type(query) == dict) and not hasattr(search, '__iter__'): multiple_queries = True num_cores = best_num_parallel(num_cores, len(query)) denom = len(query) elif hasattr(search, '__iter__') and type(search) != dict: multiple_search = True num_cores = best_num_parallel(num_cores, len(list(search.keys()))) denom = len(list(search.keys())) elif hasattr(function_filter, '__iter__'): multiple_option = True num_cores = best_num_parallel(num_cores, len(list(function_filter.keys()))) denom = len(list(function_filter.keys())) elif just_speakers: from build import get_speaker_names_from_xml_corpus multiple_speakers = True if just_speakers == 'each' or just_speakers == ['each']: just_speakers = get_speaker_names_from_xml_corpus(corpus.path) if len(just_speakers) == 0: print('No speaker name data found.') return num_cores = best_num_parallel(num_cores, len(just_speakers)) denom = len(just_speakers) if type(multiprocess) == int: num_cores = multiprocess if multiprocess is False: num_cores = 1 # make sure quicksaves are right type if quicksave is True: raise ValueError('quicksave must be string when using pmultiquery.') # the options that don't change d = { #'paralleling': True, 'function': 'interrogator', 'root': root, 'note': note, 'denominator': denom} # add kwargs to query for k, v in list(kwargs.items()): d[k] = v # make a list of dicts to pass to interrogator, # with the iterable unique in every one ds = [] if multiple_corpora: for index, p in enumerate(corpus): name = p.name a_dict = dict(d) a_dict['corpus'] = p a_dict['search'] = search a_dict['query'] = query a_dict['show'] = show a_dict['outname'] = name.replace('-parsed', '') a_dict['just_speakers'] = just_speakers a_dict['paralleling'] = index a_dict['printstatus'] = False ds.append(a_dict) elif multiple_queries: for index, (name, q) in enumerate(query.items()): a_dict = dict(d) a_dict['corpus'] = corpus a_dict['search'] = search a_dict['query'] = q a_dict['show'] = show a_dict['outname'] = name a_dict['just_speakers'] = just_speakers a_dict['paralleling'] = index a_dict['printstatus'] = False ds.append(a_dict) elif multiple_option: for index, (name, q) in enumerate(function_filter.items()): a_dict = dict(d) a_dict['corpus'] = corpus a_dict['search'] = search a_dict['query'] = query a_dict['show'] = show a_dict['outname'] = name a_dict['just_speakers'] = just_speakers a_dict['paralleling'] = index a_dict['function_filter'] = q a_dict['printstatus'] = False ds.append(a_dict) elif multiple_speakers: for index, name in enumerate(just_speakers): a_dict = dict(d) a_dict['corpus'] = corpus a_dict['search'] = search a_dict['query'] = query a_dict['show'] = show a_dict['outname'] = name a_dict['just_speakers'] = [name] a_dict['function_filter'] = function_filter a_dict['paralleling'] = index a_dict['printstatus'] = False ds.append(a_dict) elif multiple_search: for index, val in enumerate(search): a_dict = dict(d) a_dict['corpus'] = corpus a_dict['search'] = val a_dict['query'] = query a_dict['show'] = show a_dict['outname'] = name a_dict['just_speakers'] = just_speakers a_dict['function_filter'] = function_filter a_dict['paralleling'] = index a_dict['printstatus'] = False ds.append(a_dict) if kwargs.get('do_concordancing') is False: message = 'Interrogating' elif kwargs.get('do_concordancing') is True: message = 'Interrogating and concordancing' elif kwargs.get('do_concordancing').lower() == 'only': message = 'Concordancing' time = strftime("%H:%M:%S", localtime()) sformat = '' for i, (k, v) in enumerate(list(search.items())): if type(v) == list: vformat = ', '.join(v[:5]) if len(v) > 5: vformat += ' ...' else: vformat = v sformat += '%s: %s' %(k, vformat) if i < len(search.keys()) - 1: sformat += '\n ' if multiple_corpora and not multiple_option: corplist = "\n ".join([i.name for i in corpus[:20]]) if len(corpus) > 20: corplist += '\n ... and %d more ...\n' % (len(corpus) - 20) print(("\n%s: Beginning %d corpus interrogations (in %d parallel processes):\n %s" \ "\n Query: '%s'\n %s corpus ... \n" % (time, len(corpus), num_cores, corplist, sformat, message))) elif multiple_queries: print(("\n%s: Beginning %d corpus interrogations (in %d parallel processes): %s" \ "\n Queries: '%s'\n %s corpus ... \n" % (time, len(search), num_cores, corpus.name, "', '".join(list(search.values())), message) )) elif multiple_search: print(("\n%s: Beginning %d corpus interrogations (in %d parallel processes): %s" \ "\n Queries: '%s'\n %s corpus ... \n" % (time, len(list(search.keys())), num_cores, corpus.name, str(list(search.values())), message))) elif multiple_option: print(("\n%s: Beginning %d parallel corpus interrogations (multiple options): %s" \ "\n Query: '%s'\n %s corpus ... \n" % (time, num_cores, corpus.name, sformat, message) )) elif multiple_speakers: print(("\n%s: Beginning %d parallel corpus interrogations: %s" \ "\n Query: '%s'\n %s corpus ... \n" % (time, num_cores, corpus.name, sformat, message) )) # run in parallel, get either a list of tuples (non-c option) # or a dataframe (c option) #import sys #reload(sys) #stdout=sys.stdout failed = False terminal = False used_joblib = False #ds = ds[::-1] if not root: from blessings import Terminal terminal = Terminal() print('\n' * (len(ds) - 2)) for dobj in ds: linenum = dobj['paralleling'] # this try handles nosetest problems in sublime text try: with terminal.location(0, terminal.height - (linenum + 1)): # this is a really bad idea. thetime = strftime("%H:%M:%S", localtime()) num_spaces = 26 - len(dobj['outname']) print('%s: QUEUED: %s' % (thetime, dobj['outname'])) except: pass if not root and multiprocess: #res = Parallel(n_jobs=num_cores)(delayed(interrogator)(**x) for x in ds) try: #ds = sorted(ds, key=lambda k: k['paralleling'], reverse = True) res = Parallel(n_jobs=num_cores)(delayed(interrogator)(**x) for x in ds) used_joblib = True except: failed = True print('Multiprocessing failed.') raise if not res: failed = True else: res = [] for index, d in enumerate(ds): d['startnum'] = (100 / denom) * index res.append(interrogator(**d)) try: res = sorted(res) except: pass # multiprocessing way #from multiprocessing import Process #from interrogator import interrogator #jobs = [] ##for d in ds: ## p = multiprocessing.Process(target=interrogator, kwargs=(**d,)) ## jobs.append(p) ## p.start() ## while p.is_alive(): ## import time ## time.sleep(2) ## if root: ## root.update() #result_queue = multiprocessing.Queue() # #for d in ds: #funs = [interrogator(result_queue, **kwargs) for kwargs in ds] #jobs = [multiprocessing.Process(mc) for mc in funs] #for job in jobs: job.start() #for job in jobs: job.join() #results = [result_queue.get() for mc in funs] import corpkit from interrogation import Concordance if kwargs.get('do_concordancing') == 'only': concs = pd.concat([x for x in res]) thetime = strftime("%H:%M:%S", localtime()) print('\n\n%s: Finished! %d results.\n\n' % (thetime, len(concs.index))) return Concordance(concs) from collections import OrderedDict if not all(type(i.results) == pd.core.series.Series for i in res): out = OrderedDict() for interrog, d in zip(res, ds): for unpicklable in ['note', 'root']: interrog.query.pop(unpicklable, None) out[interrog.query['outname']] = interrog if quicksave: fullpath = os.path.join('saved_interrogations', quicksave) while os.path.isdir(fullpath): selection = input("\nSave error: %s already exists in %s.\n\nType 'o' to overwrite, or enter a new name: " % (quicksave, 'saved_interrogations')) if selection == 'o' or selection == 'O': import shutil shutil.rmtree(fullpath) else: import os fullpath = os.path.join('saved_interrogations', selection) for k, v in list(out.items()): save(v, k, savedir = fullpath, print_info = False) time = strftime("%H:%M:%S", localtime()) print("\n%s: %d files saved to %s" % ( time, len(list(out.keys())), fullpath)) time = strftime("%H:%M:%S", localtime()) print("\n\n%s: Finished! Output is a dictionary with keys:\n\n '%s'\n" % (time, "'\n '".join(sorted(out.keys())))) from interrogation import Interrodict return Interrodict(out) # make query and total branch, save, return else: #print sers #print ds if multiple_corpora and not mult_corp_are_subs: sers = [i.results for i in res] out = pd.DataFrame(sers, index = [i.query['outname'] for i in res]) out = out.reindex_axis(sorted(out.columns), axis=1) # sort cols out = out.fillna(0) # nan to zero out = out.astype(int) # float to int out = out.T else: out = pd.concat([r.results for r in res], axis = 1) # format like normal out = out[sorted(list(out.columns))] out = out.T out = out.fillna(0) # nan to zero out = out.astype(int) if 'c' in show and mult_corp_are_subs: out = out.sum() out.index = sorted(list(out.index)) # sort by total if type(out) == pd.core.frame.DataFrame: out.ix['Total-tmp'] = out.sum() tot = out.ix['Total-tmp'] out = out[tot.argsort()[::-1]] out = out.drop('Total-tmp', axis = 0) out = out.edit(sort_by = sort_by, print_info = False, keep_stats = False, \ df1_always_df = kwargs.get('df1_always_df')) if len(out.results.columns) == 1: out.results = out.results.sort_index() if kwargs.get('do_concordancing') is True: concs = pd.concat([x.concordance for x in res], ignore_index = True) concs = concs.sort_values(by='c') concs = concs.reset_index(drop=True) out.concordance = Concordance(concs) thetime = strftime("%H:%M:%S", localtime()) if terminal: with terminal.location(0, terminal.height): print('\n\n%s: Finished! %d unique results, %d total.%s' % (thetime, len(out.results.columns), out.totals.sum(), '\n')) else: print('\n\n%s: Finished! %d unique results, %d total.%s' % (thetime, len(out.results.columns), out.totals.sum(), '\n')) #if used_joblib: if quicksave: from other import save save(out, quicksave) print('\n') return out
def pmultiquery(corpus, search, show='words', query='any', sort_by='total', quicksave=False, multiprocess='default', just_speakers=False, root=False, note=False, print_info=True, **kwargs): """Parallel process multiple queries or corpora. This function is used by interrogator() for multiprocessing. There's no reason to call this function yourself.""" import collections import os import pandas as pd import collections from collections import namedtuple from time import strftime, localtime import corpkit from interrogator import interrogator from editor import editor from other import save from interrogation import Interrogation try: from joblib import Parallel, delayed except: pass #raise ValueError('joblib, the module used for multiprocessing, cannot be found. ' \ # 'Install with:\n\n pip install joblib') import multiprocessing locs = locals() for k, v in kwargs.items(): locs[k] = v def best_num_parallel(num_cores, num_queries): import corpkit """decide how many parallel processes to run the idea, more or less, is to balance the load when possible""" if num_queries <= num_cores: return num_queries if num_queries > num_cores: if (num_queries / num_cores) == num_cores: return int(num_cores) if num_queries % num_cores == 0: try: return max([ int(num_queries / n) for n in range(2, num_cores) if int(num_queries / n) <= num_cores ]) except ValueError: return num_cores else: import math if (float(math.sqrt(num_queries))).is_integer(): square_root = math.sqrt(num_queries) if square_root <= num_queries / num_cores: return int(square_root) return num_cores num_cores = multiprocessing.cpu_count() # what is our iterable? ... multiple_option = False multiple_queries = False multiple_speakers = False multiple_corpora = False multiple_search = False mult_corp_are_subs = False denom = 1 if hasattr(corpus, '__iter__'): multiple_corpora = True num_cores = best_num_parallel(num_cores, len(corpus)) denom = len(corpus) if all(c.__class__ == corpkit.corpus.Subcorpus for c in corpus): mult_corp_are_subs = True elif (type(query) == list or type(query) == dict) and not hasattr(search, '__iter__'): multiple_queries = True num_cores = best_num_parallel(num_cores, len(query)) denom = len(query) elif hasattr(search, '__iter__') and all( type(i) == dict for i in list(search.values())): multiple_search = True num_cores = best_num_parallel(num_cores, len(list(search.keys()))) denom = len(list(search.keys())) elif just_speakers: from build import get_speaker_names_from_xml_corpus multiple_speakers = True if just_speakers == 'each' or just_speakers == ['each']: just_speakers = get_speaker_names_from_xml_corpus(corpus.path) if len(just_speakers) == 0: print('No speaker name data found.') return num_cores = best_num_parallel(num_cores, len(just_speakers)) denom = len(just_speakers) # if this thing has already come through multiquery, don't multiprocess this time #if kwargs.get('outname'): # multiprocess = False if multiple_corpora and any(x is True for x in [ multiple_speakers, multiple_queries, multiple_search, multiple_option ]): from corpus import Corpus, Corpora if corpus.__class__ == Corpora: multiprocess = False else: corpus = Corpus(corpus) if type(multiprocess) == int: num_cores = multiprocess if multiprocess is False: num_cores = 1 # make sure quicksaves are right type if quicksave is True: raise ValueError('quicksave must be string when using pmultiquery.') # the options that don't change d = { #'paralleling': True, 'function': 'interrogator', 'root': root, 'note': note, 'denominator': denom } # add kwargs to query for k, v in list(kwargs.items()): d[k] = v # make a list of dicts to pass to interrogator, # with the iterable unique in every one ds = [] if multiple_corpora: for index, p in enumerate(corpus): name = p.name a_dict = dict(d) a_dict['corpus'] = p a_dict['search'] = search a_dict['query'] = query a_dict['show'] = show a_dict['outname'] = name.replace('-parsed', '') a_dict['just_speakers'] = just_speakers a_dict['paralleling'] = index a_dict['printstatus'] = False ds.append(a_dict) elif multiple_queries: for index, (name, q) in enumerate(query.items()): a_dict = dict(d) a_dict['corpus'] = corpus a_dict['search'] = search a_dict['query'] = q a_dict['show'] = show a_dict['outname'] = name a_dict['just_speakers'] = just_speakers a_dict['paralleling'] = index a_dict['printstatus'] = False ds.append(a_dict) elif multiple_speakers: for index, name in enumerate(just_speakers): a_dict = dict(d) a_dict['corpus'] = corpus a_dict['search'] = search a_dict['query'] = query a_dict['show'] = show a_dict['outname'] = name a_dict['just_speakers'] = [name] a_dict['paralleling'] = index a_dict['printstatus'] = False ds.append(a_dict) elif multiple_search: for index, (name, val) in enumerate(search.items()): a_dict = dict(d) a_dict['corpus'] = corpus a_dict['search'] = val a_dict['query'] = query a_dict['show'] = show a_dict['outname'] = name a_dict['just_speakers'] = just_speakers a_dict['paralleling'] = index a_dict['printstatus'] = False ds.append(a_dict) if kwargs.get('do_concordancing') is False: message = 'Interrogating' elif kwargs.get('do_concordancing') is True: message = 'Interrogating and concordancing' elif kwargs.get('do_concordancing').lower() == 'only': message = 'Concordancing' time = strftime("%H:%M:%S", localtime()) sformat = '' if multiple_queries: to_it_over = query else: to_it_over = search for i, (k, v) in enumerate(list(to_it_over.items())): if type(v) == list: vformat = ', '.join(v[:5]) if len(v) > 5: vformat += ' ...' elif type(v) == dict: vformat = '' for kk, vv in v.items(): if type(vv) == list: vv = ', '.join(vv[:5]) vformat += '\n %s: %s' % (kk, vv) if len(vv) > 5: vformat += ' ...' else: vformat = v sformat += '%s: %s' % (k, vformat) if i < len(to_it_over.keys()) - 1: sformat += '\n ' if print_info: if multiple_corpora and not multiple_option: corplist = "\n ".join([i.name for i in corpus[:20]]) if len(corpus) > 20: corplist += '\n ... and %d more ...\n' % (len(corpus) - 20) print(("\n%s: Beginning %d corpus interrogations (in %d parallel processes):\n %s" \ "\n Query: %s\n %s corpus ... \n" % (time, len(corpus), num_cores, corplist, sformat, message))) elif multiple_queries: print(("\n%s: Beginning %d corpus interrogations (in %d parallel processes): %s" \ "\n Queries: %s\n %s corpus ... \n" % (time, len(query), num_cores, corpus.name, sformat, message) )) elif multiple_search: print(("\n%s: Beginning %d corpus interrogations (in %d parallel processes): %s" \ "\n Queries: %s\n %s corpus ... \n" % (time, len(list(search.keys())), num_cores, corpus.name, sformat, message))) elif multiple_option: print(("\n%s: Beginning %d parallel corpus interrogations (multiple options): %s" \ "\n Query: %s\n %s corpus ... \n" % (time, num_cores, corpus.name, sformat, message) )) elif multiple_speakers: print(("\n%s: Beginning %d parallel corpus interrogations: %s" \ "\n Query: %s\n %s corpus ... \n" % (time, num_cores, corpus.name, sformat, message) )) # run in parallel, get either a list of tuples (non-c option) # or a dataframe (c option) #import sys #reload(sys) #stdout=sys.stdout failed = False terminal = False used_joblib = False #ds = ds[::-1] if not root and print_info: from blessings import Terminal terminal = Terminal() print('\n' * (len(ds) - 2)) for dobj in ds: linenum = dobj['paralleling'] # this try handles nosetest problems in sublime text try: with terminal.location(0, terminal.height - (linenum + 1)): # this is a really bad idea. thetime = strftime("%H:%M:%S", localtime()) num_spaces = 26 - len(dobj['outname']) print('%s: QUEUED: %s' % (thetime, dobj['outname'])) except: pass if not root and multiprocess: #res = Parallel(n_jobs=num_cores)(delayed(interrogator)(**x) for x in ds) try: #ds = sorted(ds, key=lambda k: k['paralleling'], reverse = True) res = Parallel(n_jobs=num_cores)(delayed(interrogator)(**x) for x in ds) used_joblib = True except: failed = True print('Multiprocessing failed.') raise if not res: failed = True else: res = [] for index, d in enumerate(ds): d['startnum'] = (100 / denom) * index res.append(interrogator(**d)) try: res = sorted(res) except: pass # multiprocessing way #from multiprocessing import Process #from interrogator import interrogator #jobs = [] ##for d in ds: ## p = multiprocessing.Process(target=interrogator, kwargs=(**d,)) ## jobs.append(p) ## p.start() ## while p.is_alive(): ## import time ## time.sleep(2) ## if root: ## root.update() #result_queue = multiprocessing.Queue() # #for d in ds: #funs = [interrogator(result_queue, **kwargs) for kwargs in ds] #jobs = [multiprocessing.Process(mc) for mc in funs] #for job in jobs: job.start() #for job in jobs: job.join() #results = [result_queue.get() for mc in funs] import corpkit from interrogation import Concordance if kwargs.get('do_concordancing') == 'only': concs = pd.concat([x for x in res]) thetime = strftime("%H:%M:%S", localtime()) if print_info: print('\n\n%s: Finished! %d results.\n\n' % (thetime, len(concs.index))) return Concordance(concs) from collections import OrderedDict if not all(type(i.results) == pd.core.series.Series for i in res): out = OrderedDict() for interrog, d in zip(res, ds): for unpicklable in ['note', 'root']: interrog.query.pop(unpicklable, None) try: out[interrog.query['outname']] = interrog except KeyError: out[d['outname']] = interrog if quicksave: fullpath = os.path.join('saved_interrogations', quicksave) while os.path.isdir(fullpath): selection = input( "\nSave error: %s already exists in %s.\n\nType 'o' to overwrite, or enter a new name: " % (quicksave, 'saved_interrogations')) if selection == 'o' or selection == 'O': import shutil shutil.rmtree(fullpath) else: import os fullpath = os.path.join('saved_interrogations', selection) for k, v in list(out.items()): save(v, k, savedir=fullpath, print_info=False) time = strftime("%H:%M:%S", localtime()) print("\n%s: %d files saved to %s" % (time, len(list(out.keys())), fullpath)) time = strftime("%H:%M:%S", localtime()) if print_info: print( "\n\n%s: Finished! Output is a dictionary with keys:\n\n '%s'\n" % (time, "'\n '".join(sorted(out.keys())))) from interrogation import Interrodict idict = Interrodict(out) # remove unpicklable bits from query from types import ModuleType, FunctionType, BuiltinMethodType, BuiltinFunctionType locs = {k: v for k, v in locs.items() if not isinstance(v, ModuleType) \ and not isinstance(v, FunctionType) \ and not isinstance(v, BuiltinFunctionType) \ and not isinstance(v, BuiltinMethodType)} idict.query = locs return idict # make query and total branch, save, return else: #print sers #print ds if multiple_corpora and not mult_corp_are_subs: sers = [i.results for i in res] out = pd.DataFrame(sers, index=[i.query['outname'] for i in res]) out = out.reindex_axis(sorted(out.columns), axis=1) # sort cols out = out.fillna(0) # nan to zero out = out.astype(int) # float to int out = out.T else: try: out = pd.concat([r.results for r in res], axis=1) except ValueError: return None # format like normal out = out[sorted(list(out.columns))] out = out.T out = out.fillna(0) # nan to zero out = out.astype(int) if 'c' in show and mult_corp_are_subs: out = out.sum() out.index = sorted(list(out.index)) # sort by total if type(out) == pd.core.frame.DataFrame: out.ix['Total-tmp'] = out.sum() tot = out.ix['Total-tmp'] out = out[tot.argsort()[::-1]] out = out.drop('Total-tmp', axis=0) out = out.edit(sort_by = sort_by, print_info = False, keep_stats = False, \ df1_always_df = kwargs.get('df1_always_df')) if len(out.results.columns) == 1: out.results = out.results.sort_index() if kwargs.get('do_concordancing') is True: concs = pd.concat([x.concordance for x in res], ignore_index=True) concs = concs.sort_values(by='c') concs = concs.reset_index(drop=True) out.concordance = Concordance(concs) thetime = strftime("%H:%M:%S", localtime()) if terminal and print_info: with terminal.location(0, terminal.height): print('\n\n%s: Finished! %d unique results, %d total.%s' % (thetime, len( out.results.columns), out.totals.sum(), '\n')) else: if print_info: print('\n\n%s: Finished! %d unique results, %d total.%s' % (thetime, len( out.results.columns), out.totals.sum(), '\n')) #if used_joblib: if quicksave: from other import save save(out, quicksave) return out
class FractionTaxaBarStack(Graph): short_name = 'fraction_taxa_barstack' bottom = 0.4 top = 0.95 left = 0.1 right = 0.95 formats = ('pdf', 'eps') def plot(self): # Make Frame # self.frame = OrderedDict( (('%s - %s' % (p, f), getattr(p.fractions, f).rdp.phyla) for f in ('low', 'med', 'big') for p in self.parent.pools)) self.frame = pandas.DataFrame(self.frame) self.frame = self.frame.fillna(0) # Rename # new_names = { u"run001-pool01 - low": "2-step PCR low", u"run001-pool02 - low": "2-step PCR low", u"run001-pool03 - low": "2-step PCR low", u"run001-pool04 - low": "1-step PCR low", u"run002-pool01 - low": "New chem low", u"run001-pool01 - med": "2-step PCR med", u"run001-pool02 - med": "2-step PCR med", u"run001-pool03 - med": "2-step PCR med", u"run001-pool04 - med": "1-step PCR med", u"run002-pool01 - med": "New chem med", u"run001-pool01 - big": "2-step PCR high", u"run001-pool02 - big": "2-step PCR high", u"run001-pool03 - big": "2-step PCR high", u"run001-pool04 - big": "1-step PCR high", u"run002-pool01 - big": "New chem high", } self.frame.rename(columns=new_names, inplace=True) self.frame = self.frame.transpose() # Group low abundant into 'others' # low_abundance = self.frame.sum() < 30000 other_count = self.frame.loc[:, low_abundance].sum(axis=1) self.frame = self.frame.loc[:, ~low_abundance] self.frame['Others'] = other_count # Normalize # self.frame = self.frame.apply(lambda x: 100 * x / x.sum(), axis=1) # Sort the table by sum # sums = self.frame.sum() sums.sort(ascending=False) self.frame = self.frame.reindex_axis(sums.keys(), axis=1) # Plot # fig = pyplot.figure() axes = self.frame.plot(kind='bar', stacked=True, color=cool_colors) fig = pyplot.gcf() # Other # axes.set_ylabel('Relative abundances in percent') axes.xaxis.grid(False) axes.yaxis.grid(False) axes.set_ylim([0, 100]) # Put a legend below current axis axes.legend(loc='upper center', bbox_to_anchor=(0.5, -0.40), fancybox=True, shadow=True, ncol=5, prop={'size': 10}) # Font size # axes.tick_params(axis='x', which='major', labelsize=11) # Save it # self.save_plot(fig, axes) self.frame.to_csv(self.csv_path) pyplot.close(fig)
def pmultiquery(corpus, search, show='words', query='any', sort_by='total', save=False, multiprocess='default', root=False, note=False, print_info=True, subcorpora=False, **kwargs ): """ - Parallel process multiple queries or corpora. - This function is used by corpkit.interrogator.interrogator() - for multiprocessing. - There's no reason to call this function yourself. """ import os from pandas import DataFrame, Series import pandas as pd import collections from collections import namedtuple, OrderedDict from time import strftime, localtime import corpkit from corpkit.interrogator import interrogator from corpkit.interrogation import Interrogation, Interrodict from corpkit.process import canpickle try: from joblib import Parallel, delayed except ImportError: pass import multiprocessing locs = locals() for k, v in kwargs.items(): locs[k] = v in_notebook = locs.get('in_notebook') def best_num_parallel(num_cores, num_queries): """decide how many parallel processes to run the idea, more or less, is to balance the load when possible""" import corpkit if num_queries <= num_cores: return num_queries if num_queries > num_cores: if (num_queries / num_cores) == num_cores: return int(num_cores) if num_queries % num_cores == 0: try: return max([int(num_queries / n) for n in range(2, num_cores) \ if int(num_queries / n) <= num_cores]) except ValueError: return num_cores else: import math if (float(math.sqrt(num_queries))).is_integer(): square_root = math.sqrt(num_queries) if square_root <= num_queries / num_cores: return int(square_root) return num_cores num_cores = multiprocessing.cpu_count() # what is our iterable? ... multiple = kwargs.get('multiple', False) mult_corp_are_subs = False if hasattr(corpus, '__iter__'): if all(getattr(x, 'level', False) == 's' for x in corpus): mult_corp_are_subs = True non_first_sub = None if subcorpora: non_first_sub = subcorpora[1:] if isinstance(subcorpora, list) else None subval = subcorpora if not non_first_sub else subcorpora[0] #print(subcorpora, non_first_sub, subval) if subcorpora is True: import re subcorpora = re.compile(r'.*') else: # strange travis error happened here subcorpora = corpus.metadata['fields'][subval] if len(subcorpora) == 0: print('No %s metadata found.' % str(subval)) return mapcores = {'datalist': [corpus, 'corpus'], 'multiplecorpora': [corpus, 'corpus'], 'namedqueriessingle': [query, 'query'], 'namedqueriesmultiple': [search, 'search'], 'subcorpora': [subcorpora, 'subcorpora']} # a is a dummy, just to produce default one toiter, itsname = mapcores.get(multiple, [False, False]) if isinstance(toiter, dict): toiter = toiter.items() denom = len(toiter) num_cores = best_num_parallel(num_cores, denom) # todo: code below makes no sense vals = ['eachspeaker', 'multiplespeaker', 'namedqueriesmultiple'] if multiple == 'multiplecorpora' and any(x is True for x in vals): from corpkit.corpus import Corpus, Corpora if isinstance(corpus, Corpora): multiprocess = False else: corpus = Corpus(corpus) if isinstance(multiprocess, int): num_cores = multiprocess if multiprocess is False: num_cores = 1 # make sure saves are right type if save is True: raise ValueError('save must be string when multiprocessing.') # make a list of dicts to pass to interrogator, # with the iterable unique in every one locs['printstatus'] = False locs['multiprocess'] = False locs['df1_always_df'] = False locs['files_as_subcorpora'] = False locs['corpus'] = corpus if multiple == 'multiplespeaker': locs['multispeaker'] = True if isinstance(non_first_sub, list) and len(non_first_sub) == 1: non_first_sub = non_first_sub[0] # make the default query locs = {k: v for k, v in locs.items() if canpickle(v)} # make a new dict for every iteration ds = [dict(**locs) for i in range(denom)] for index, (d, bit) in enumerate(zip(ds, toiter)): d['paralleling'] = index if multiple in ['namedqueriessingle', 'namedqueriesmultiple']: d[itsname] = bit[1] d['outname'] = bit[0] elif multiple in ['multiplecorpora', 'datalist']: d['outname'] = bit.name.replace('-parsed', '') d[itsname] = bit elif multiple in ['subcorpora']: d[itsname] = bit jmd = {subval: bit} # put this earlier j2 = kwargs.get('just_metadata', False) if not j2: j2 = {} jmd.update(j2) d['just_metadata'] = jmd d['outname'] = bit d['by_metadata'] = False d['subcorpora'] = non_first_sub if non_first_sub: d['print_info'] = False # message printer should be a function... if kwargs.get('conc') is False: message = 'Interrogating' elif kwargs.get('conc') is True: message = 'Interrogating and concordancing' elif kwargs.get('conc').lower() == 'only': message = 'Concordancing' time = strftime("%H:%M:%S", localtime()) from corpkit.process import dictformat if print_info: # proper printing for plurals # in truth this needs to be revised, it's horrible. sformat = dictformat(search, query) if num_cores == 1: add_es = '' else: add_es = 'es' if multiple in ['multiplecorpora', 'datalist']: corplist = "\n ".join([i.name for i in list(corpus)[:20]]) if len(corpus) > 20: corplist += '\n ... and %d more ...\n' % (len(corpus) - 20) print(("\n%s: Beginning %d corpus interrogations (in %d parallel process%s):\n %s" \ "\n Query: %s\n %s corpus ... \n" % (time, len(corpus), num_cores, add_es, corplist, sformat, message))) elif multiple == 'namedqueriessingle': print(("\n%s: Beginning %d corpus interrogations (in %d parallel process%s): %s" \ "\n Queries: %s\n %s corpus ... \n" % (time, len(query), num_cores, add_es, corpus.name, sformat, message) )) elif multiple == 'namedqueriesmultiple': print(("\n%s: Beginning %d corpus interrogations (in %d parallel process%s): %s" \ "\n Queries: %s\n %s corpus ... \n" % (time, len(list(search.keys())), num_cores, add_es, corpus.name, sformat, message))) elif multiple in ['eachspeaker', 'multiplespeaker']: print(("\n%s: Beginning %d parallel corpus interrogation%s: %s" \ "\n Query: %s\n %s corpus ... \n" % (time, num_cores, add_es.lstrip('e'), corpus.name, sformat, message) )) elif multiple in ['subcorpora']: print(("\n%s: Beginning %d parallel corpus interrogation%s: %s" \ "\n Query: %s\n %s corpus ... \n" % (time, num_cores, add_es.lstrip('e'), corpus.name, sformat, message) )) # run in parallel, get either a list of tuples (non-c option) # or a dataframe (c option) #import sys #reload(sys) #stdout=sys.stdout failed = False terminal = False used_joblib = False #ds = ds[::-1] #todo: the number of blank lines to print can be way wrong if not root and print_info: from blessings import Terminal terminal = Terminal() print('\n' * (len(ds) - 2)) for dobj in ds: linenum = dobj['paralleling'] # this try handles nosetest problems in sublime text try: with terminal.location(0, terminal.height - (linenum + 1)): # this is a really bad idea. thetime = strftime("%H:%M:%S", localtime()) num_spaces = 26 - len(dobj['outname']) print('%s: QUEUED: %s' % (thetime, dobj['outname'])) except: pass if not root and multiprocess: try: res = Parallel(n_jobs=num_cores)(delayed(interrogator)(**x) for x in ds) used_joblib = True except: failed = True print('Multiprocessing failed.') raise if not res: failed = True else: res = [] for index, d in enumerate(ds): d['startnum'] = (100 / denom) * index res.append(interrogator(**d)) try: res = sorted([i for i in res if i]) except: pass # remove unpicklable bits from query from types import ModuleType, FunctionType, BuiltinMethodType, BuiltinFunctionType badtypes = (ModuleType, FunctionType, BuiltinFunctionType, BuiltinMethodType) qlocs = {k: v for k, v in locs.items() if not isinstance(v, badtypes)} if hasattr(qlocs.get('corpus', False), 'name'): qlocs['corpus'] = qlocs['corpus'].path else: qlocs['corpus'] = list([i.path for i in qlocs.get('corpus', [])]) # return just a concordance from corpkit.interrogation import Concordance if kwargs.get('conc') == 'only': concs = pd.concat([x for x in res]) thetime = strftime("%H:%M:%S", localtime()) concs = concs.reset_index(drop=True) if kwargs.get('maxconc'): concs = concs[:kwargs.get('maxconc')] lines = Concordance(concs) if save: lines.save(save, print_info=print_info) if print_info: print('\n\n%s: Finished! %d results.\n\n' % (thetime, format(len(concs.index), ','))) return lines # return interrodict (to become multiindex) if isinstance(res[0], Interrodict) or not all(isinstance(i.results, Series) for i in res): out = OrderedDict() for interrog, d in zip(res, ds): for unpicklable in ['note', 'root']: interrog.query.pop(unpicklable, None) try: out[interrog.query['outname']] = interrog except KeyError: out[d['outname']] = interrog idict = Interrodict(out) if print_info: thetime = strftime("%H:%M:%S", localtime()) print("\n\n%s: Finished! Output is multi-indexed." % thetime) idict.query = qlocs if save: idict.save(save, print_info=print_info) return idict # make query and total branch, save, return # todo: standardise this so we don't have to guess transposes # else: if multiple == 'multiplecorpora' and not mult_corp_are_subs: sers = [i.results for i in res] out = DataFrame(sers, index=[i.query['outname'] for i in res]) out = out.reindex_axis(sorted(out.columns), axis=1) # sort cols out = out.fillna(0) # nan to zero out = out.astype(int) # float to int out = out.T else: # make a series from counts if all(len(i.results) == 1 for i in res): out = pd.concat([r.results for r in res]) out = out.sort_index() else: try: out = pd.concat([r.results for r in res], axis=1) out = out.T out.index = [i.query['outname'] for i in res] except ValueError: return None # format like normal # this sorts subcorpora, which are cls out = out[sorted(list(out.columns))] # puts subcorpora in the right place if not mult_corp_are_subs and multiple != 'subcorpora': out = out.T if multiple == 'subcorpora': out = out.sort_index() out = out.fillna(0) # nan to zero out = out.astype(int) if 'c' in show and mult_corp_are_subs: out = out.sum() out.index = sorted(list(out.index)) # sort by total if isinstance(out, DataFrame): out = out[list(out.sum().sort_values(ascending=False).index)] # really need to figure out the deal with tranposing! if all(x.endswith('.xml') for x in list(out.columns)) \ or all(x.endswith('.txt') for x in list(out.columns)) \ or all(x.endswith('.conll') for x in list(out.columns)): out = out.T if kwargs.get('nosubmode'): out = out.sum() from corpkit.interrogation import Interrogation tt = out.sum(axis=1) if isinstance(out, DataFrame) else out.sum() out = Interrogation(results=out, totals=tt, query=qlocs) if hasattr(out, 'columns') and len(out.columns) == 1: out = out.sort_index() if kwargs.get('conc') is True: try: concs = pd.concat([x.concordance for x in res], ignore_index=True) concs = concs.sort_values(by='c') concs = concs.reset_index(drop=True) if kwargs.get('maxconc'): concs = concs[:kwargs.get('maxconc')] out.concordance = Concordance(concs) except ValueError: out.concordance = None thetime = strftime("%H:%M:%S", localtime()) if terminal: print(terminal.move(terminal.height-1, 0)) if print_info: if terminal: print(terminal.move(terminal.height-1, 0)) if hasattr(out.results, 'columns'): print('%s: Interrogation finished! %s unique results, %s total.' % (thetime, format(len(out.results.columns), ','), format(out.totals.sum(), ','))) else: print('%s: Interrogation finished! %s matches.' % (thetime, format(tt, ','))) if save: out.save(save, print_info = print_info) if list(out.results.index) == ['0'] and not kwargs.get('df1_always_df'): out.results = out.results.ix[0].sort_index() return out
class FractionTaxaBarStack(Graph): """This is figure 3 of the paper""" short_name = 'fraction_taxa_barstack' bottom = 0.4 top = 0.95 left = 0.1 right = 0.95 formats = ('pdf', 'eps') def plot(self): # Make Frame # self.frame = OrderedDict((('%s - %s' % (p,f), getattr(p.fractions, f).rdp.phyla) for f in ('low', 'med', 'big') for p in self.parent.pools)) self.frame = pandas.DataFrame(self.frame) self.frame = self.frame.fillna(0) # Rename # new_names = { u"run001-pool01 - low": "2-step PCR low", u"run001-pool02 - low": "2-step PCR low", u"run001-pool03 - low": "2-step PCR low", u"run001-pool04 - low": "1-step PCR low", u"run002-pool01 - low": "New chem low", u"run001-pool01 - med": "2-step PCR med", u"run001-pool02 - med": "2-step PCR med", u"run001-pool03 - med": "2-step PCR med", u"run001-pool04 - med": "1-step PCR med", u"run002-pool01 - med": "New chem med", u"run001-pool01 - big": "2-step PCR high", u"run001-pool02 - big": "2-step PCR high", u"run001-pool03 - big": "2-step PCR high", u"run001-pool04 - big": "1-step PCR high", u"run002-pool01 - big": "New chem high", } self.frame.rename(columns=new_names, inplace=True) self.frame = self.frame.transpose() # Group low abundant into 'others' # low_abundance = self.frame.sum() < 30000 other_count = self.frame.loc[:, low_abundance].sum(axis=1) self.frame = self.frame.loc[:, ~low_abundance] self.frame['Others'] = other_count # Normalize # self.frame = self.frame.apply(lambda x: 100*x/x.sum(), axis=1) # Sort the table by sum # sums = self.frame.sum() sums.sort(ascending=False) self.frame = self.frame.reindex_axis(sums.keys(), axis=1) # Plot # fig = pyplot.figure() axes = self.frame.plot(kind='bar', stacked=True, color=cool_colors) fig = pyplot.gcf() # Other # axes.set_ylabel('Relative abundances in percent') axes.xaxis.grid(False) axes.yaxis.grid(False) axes.set_ylim([0,100]) # Put a legend below current axis axes.legend(loc='upper center', bbox_to_anchor=(0.5, -0.40), fancybox=True, shadow=True, ncol=5, prop={'size':10}) # Font size # axes.tick_params(axis='x', which='major', labelsize=11) # Save it # self.save_plot(fig, axes) self.frame.to_csv(self.csv_path) pyplot.close(fig)
def pmultiquery(corpus, search, show='words', query='any', sort_by='total', save=False, multiprocess='default', just_speakers=False, root=False, note=False, print_info=True, **kwargs ): """ - Parallel process multiple queries or corpora. - This function is used by corpkit.interrogator.interrogator() - for multiprocessing. - There's no reason to call this function yourself.""" import os from pandas import DataFrame, Series import pandas as pd import collections from collections import namedtuple, OrderedDict from time import strftime, localtime import corpkit from corpkit.interrogator import interrogator from corpkit.interrogation import Interrogation try: from joblib import Parallel, delayed except ImportError: pass import multiprocessing locs = locals() for k, v in kwargs.items(): locs[k] = v in_notebook = locs.get('in_notebook') def best_num_parallel(num_cores, num_queries): """decide how many parallel processes to run the idea, more or less, is to balance the load when possible""" import corpkit if num_queries <= num_cores: return num_queries if num_queries > num_cores: if (num_queries / num_cores) == num_cores: return int(num_cores) if num_queries % num_cores == 0: try: return max([int(num_queries / n) for n in range(2, num_cores) \ if int(num_queries / n) <= num_cores]) except ValueError: return num_cores else: import math if (float(math.sqrt(num_queries))).is_integer(): square_root = math.sqrt(num_queries) if square_root <= num_queries / num_cores: return int(square_root) return num_cores num_cores = multiprocessing.cpu_count() # what is our iterable? ... multiple_option = False multiple_queries = False multiple_speakers = False multiple_corpora = False multiple_search = False mult_corp_are_subs = False denom = 1 if hasattr(corpus, '__iter__'): multiple_corpora = True num_cores = best_num_parallel(num_cores, len(corpus)) denom = len(corpus) if all(c.__class__ == corpkit.corpus.Subcorpus for c in corpus): mult_corp_are_subs = True elif (isinstance(query, (list, dict)) and not hasattr(search, '__iter__')): multiple_queries = True num_cores = best_num_parallel(num_cores, len(query)) denom = len(query) elif hasattr(search, '__iter__') and all(isinstance(i, dict) for i in list(search.values())): multiple_search = True num_cores = best_num_parallel(num_cores, len(list(search.keys()))) denom = len(list(search.keys())) elif just_speakers: from build import get_speaker_names_from_xml_corpus multiple_speakers = True if just_speakers == 'each' or just_speakers == ['each']: just_speakers = get_speaker_names_from_xml_corpus(corpus.path) if len(just_speakers) == 0: print('No speaker name data found.') return num_cores = best_num_parallel(num_cores, len(just_speakers)) denom = len(just_speakers) if multiple_corpora and any(x is True for x in [multiple_speakers, multiple_queries, multiple_search, multiple_option]): from corpkit.corpus import Corpus, Corpora if isinstance(corpus, Corpora): multiprocess = False else: corpus = Corpus(corpus) if isinstance(multiprocess, int): num_cores = multiprocess if multiprocess is False: num_cores = 1 # make sure saves are right type if save is True: raise ValueError('save must be string when multiprocessing.') # the options that don't change d = {'function': 'interrogator', 'root': root, 'note': note, 'denominator': denom} # add kwargs to query for k, v in list(kwargs.items()): d[k] = v # make a list of dicts to pass to interrogator, # with the iterable unique in every one ds = [] if multiple_corpora: for index, p in enumerate(corpus): name = p.name a_dict = dict(d) a_dict['corpus'] = p a_dict['search'] = search a_dict['query'] = query a_dict['show'] = show a_dict['outname'] = name.replace('-parsed', '') a_dict['just_speakers'] = just_speakers a_dict['paralleling'] = index a_dict['printstatus'] = False ds.append(a_dict) elif multiple_queries: for index, (name, q) in enumerate(query.items()): a_dict = dict(d) a_dict['corpus'] = corpus a_dict['search'] = search a_dict['query'] = q a_dict['show'] = show a_dict['outname'] = name a_dict['just_speakers'] = just_speakers a_dict['paralleling'] = index a_dict['printstatus'] = False ds.append(a_dict) elif multiple_speakers: for index, name in enumerate(just_speakers): a_dict = dict(d) a_dict['corpus'] = corpus a_dict['search'] = search a_dict['query'] = query a_dict['show'] = show a_dict['outname'] = name a_dict['just_speakers'] = [name] a_dict['paralleling'] = index a_dict['printstatus'] = False ds.append(a_dict) elif multiple_search: for index, (name, val) in enumerate(search.items()): a_dict = dict(d) a_dict['corpus'] = corpus a_dict['search'] = val a_dict['query'] = query a_dict['show'] = show a_dict['outname'] = name a_dict['just_speakers'] = just_speakers a_dict['paralleling'] = index a_dict['printstatus'] = False ds.append(a_dict) if kwargs.get('conc') is False: message = 'Interrogating' elif kwargs.get('conc') is True: message = 'Interrogating and concordancing' elif kwargs.get('conc').lower() == 'only': message = 'Concordancing' time = strftime("%H:%M:%S", localtime()) sformat = '' if multiple_queries: to_it_over = query else: to_it_over = search for i, (k, v) in enumerate(list(to_it_over.items())): if isinstance(v, list): vformat = ', '.join(v[:5]) if len(v) > 5: vformat += ' ...' elif isinstance(v, dict): vformat = '' for kk, vv in v.items(): if isinstance(vv, list): vv = ', '.join(vv[:5]) vformat += '\n %s: %s' % (kk, vv) if len(vv) > 5: vformat += ' ...' else: try: vformat = v.pattern except AttributeError: vformat = v sformat += '%s: %s' %(k, vformat) if i < len(to_it_over.keys()) - 1: sformat += '\n ' if print_info: # proper printing for plurals # in truth this needs to be revised, it's horrible. if num_cores == 1: add_es = '' else: add_es = 'es' if multiple_corpora and not multiple_option: corplist = "\n ".join([i.name for i in corpus[:20]]) if len(corpus) > 20: corplist += '\n ... and %d more ...\n' % (len(corpus) - 20) print(("\n%s: Beginning %d corpus interrogations (in %d parallel process%s):\n %s" \ "\n Query: %s\n %s corpus ... \n" % (time, len(corpus), num_cores, add_es, corplist, sformat, message))) elif multiple_queries: print(("\n%s: Beginning %d corpus interrogations (in %d parallel process%s): %s" \ "\n Queries: %s\n %s corpus ... \n" % (time, len(query), num_cores, add_es, corpus.name, sformat, message) )) elif multiple_search: print(("\n%s: Beginning %d corpus interrogations (in %d parallel process%s): %s" \ "\n Queries: %s\n %s corpus ... \n" % (time, len(list(search.keys())), num_cores, add_es, corpus.name, sformat, message))) elif multiple_option: print(("\n%s: Beginning %d parallel corpus interrogation%s (multiple options): %s" \ "\n Query: %s\n %s corpus ... \n" % (time, num_cores, add_es.lstrip('e'), corpus.name, sformat, message) )) elif multiple_speakers: print(("\n%s: Beginning %d parallel corpus interrogation%s: %s" \ "\n Query: %s\n %s corpus ... \n" % (time, num_cores, add_es.lstrip('e'), corpus.name, sformat, message) )) # run in parallel, get either a list of tuples (non-c option) # or a dataframe (c option) #import sys #reload(sys) #stdout=sys.stdout failed = False terminal = False used_joblib = False #ds = ds[::-1] if not root and print_info: from blessings import Terminal terminal = Terminal() print('\n' * (len(ds) - 2)) for dobj in ds: linenum = dobj['paralleling'] # this try handles nosetest problems in sublime text try: with terminal.location(0, terminal.height - (linenum + 1)): # this is a really bad idea. thetime = strftime("%H:%M:%S", localtime()) num_spaces = 26 - len(dobj['outname']) print('%s: QUEUED: %s' % (thetime, dobj['outname'])) except: pass if not root and multiprocess: #res = Parallel(n_jobs=num_cores)(delayed(interrogator)(**x) for x in ds) try: #ds = sorted(ds, key=lambda k: k['paralleling'], reverse = True) res = Parallel(n_jobs=num_cores)(delayed(interrogator)(**x) for x in ds) used_joblib = True except: failed = True print('Multiprocessing failed.') raise if not res: failed = True else: res = [] for index, d in enumerate(ds): d['startnum'] = (100 / denom) * index res.append(interrogator(**d)) try: res = sorted([i for i in res if i]) except: pass # remove unpicklable bits from query from types import ModuleType, FunctionType, BuiltinMethodType, BuiltinFunctionType badtypes = (ModuleType, FunctionType, BuiltinFunctionType, BuiltinMethodType) qlocs = {k: v for k, v in locs.items() if not isinstance(v, badtypes)} if hasattr(qlocs['corpus'], 'name'): qlocs['corpus'] = qlocs['corpus'].path else: qlocs['corpus'] = list([i.path for i in qlocs['corpus']]) from corpkit.interrogation import Concordance if kwargs.get('conc') == 'only': concs = pd.concat([x for x in res]) thetime = strftime("%H:%M:%S", localtime()) concs = concs.reset_index(drop=True) lines = Concordance(concs) if save: lines.save(save, print_info=print_info) if print_info: print('\n\n%s: Finished! %d results.\n\n' % (thetime, len(concs.index))) return lines if not all(isinstance(i.results, Series) for i in res): out = OrderedDict() for interrog, d in zip(res, ds): for unpicklable in ['note', 'root']: interrog.query.pop(unpicklable, None) try: out[interrog.query['outname']] = interrog except KeyError: out[d['outname']] = interrog from corpkit.interrogation import Interrodict idict = Interrodict(out) if print_info: time = strftime("%H:%M:%S", localtime()) print("\n\n%s: Finished! Output is a dictionary with keys:\n\n '%s'\n" % \ (time, "'\n '".join(sorted(out.keys())))) idict.query = qlocs if save: idict.save(save, print_info=print_info) return idict # make query and total branch, save, return # todo: standardise this so we don't have to guess transposes else: if multiple_corpora and not mult_corp_are_subs: sers = [i.results for i in res] out = DataFrame(sers, index=[i.query['outname'] for i in res]) out = out.reindex_axis(sorted(out.columns), axis=1) # sort cols out = out.fillna(0) # nan to zero out = out.astype(int) # float to int out = out.T else: try: out = pd.concat([r.results for r in res], axis=1) out = out.T out.index = [i.query['outname'] for i in res] except ValueError: return None # format like normal # this sorts subcorpora, which are cls out = out[sorted(list(out.columns))] # puts subcorpora in the right place if not mult_corp_are_subs: out = out.T out = out.fillna(0) # nan to zero out = out.astype(int) if 'c' in show and mult_corp_are_subs: out = out.sum() out.index = sorted(list(out.index)) # sort by total if isinstance(out, DataFrame): out = out[list(out.sum().sort_values(ascending=False).index)] # really need to figure out the deal with tranposing! if all(x.endswith('.xml') for x in list(out.columns)) \ or all(x.endswith('.txt') for x in list(out.columns)): out = out.T out = out.edit(sort_by=sort_by, print_info=False, keep_stats=False, \ df1_always_df=kwargs.get('df1_always_df')) out.query = qlocs if len(out.results.columns) == 1: out.results = out.results.sort_index() if kwargs.get('conc') is True: concs = pd.concat([x.concordance for x in res], ignore_index=True) concs = concs.sort_values(by='c') concs = concs.reset_index(drop=True) out.concordance = Concordance(concs) thetime = strftime("%H:%M:%S", localtime()) if terminal and print_info: with terminal.location(0, terminal.height): print('\n\n%s: Finished! %d unique results, %d total.%s' % (thetime, len(out.results.columns), out.totals.sum(), '\n')) else: if print_info: print('\n\n%s: Finished! %d unique results, %d total.%s' % (thetime, len(out.results.columns), out.totals.sum(), '\n')) if save: out.save(save, print_info = print_info) return out