def to_rgba(color,alpha): """ Converts from hex|rgb to rgba Parameters: ----------- color : string Color representation on hex or rgb alpha : float Value from 0 to 1.0 that represents the alpha value. Example: to_rgba('#E1E5ED',0.6) to_rgba('#f03',0.7) to_rgba('rgb(23,23,23)',.5) """ color=color.lower() if 'rgba' in color: cl=list(eval(color.replace('rgba',''))) cl[3]=alpha return 'rgba'+str(tuple(cl)) elif 'rgb' in color: r,g,b=eval(color.replace('rgb','')) return 'rgba'+str((r,g,b,alpha)) else: return to_rgba(hex_to_rgb(color),alpha)
def run(ast, args=['<progname>'], check_types=False): ast = rewrite(ast, replace_nodes) log.final_ast("the final AST is:\n", ast) frame = Frame() ast.eval(frame) log.topframe("the top frame is\n", frame) if 'main' not in frame: print("no main function defined, exiting") return 0 # type inference if check_types: with frame as newframe: newframe['argc'] = Int(len(args)) newframe['argv'] = Array(map(Str, args)) main = newframe['main'] main.infer_type(newframe) assert main.type.ret == Int, \ "main() should return Int but got %s" % main.type.ret with frame as newframe: newframe['argc'] = Int(len(args)) newframe['argv'] = Array(map(Str, args)) r = newframe['main'].Call(newframe) if isinstance(r, Int): return r.to_int() else: return 0
def __init__(self): _pos = file("pos", "r") _neg = file("neg", "r") self.pos = eval(_pos.readline().strip()) self.neg = eval(_neg.readline().strip()) _pos.close() _neg.close()
def add_signed_footer(message=None,ifolder=None,tfile=None,ext='all',rewrite=False): from sq_tools import add_signed_footer as sign from ast import literal_eval as eval if ext=='all': ext=['nsp','nsx','nsz','xci','xcz'] else: if isinstance(ext, list): ext=ext else: try: ext=eval(ext) except:pass ext=ext.split(',') if ifolder!=None: try: ifolder=eval(ifolder) except:pass files=listmanager.folder_to_list(ifolder,ext) elif tfile!=None: try: tfile=eval(tfile) except:pass files=read_lines_to_list(tfile,all=True) else: return False for filepath in files: try: sign(filepath,message,rewrite) except: print('Error in '+filepath)
def register(): form = RegisterForm() types = [type(field).__name__ for field in form] if form.validate_on_submit(): registration_statuses = eval(REGISTRATION["registration_statuses"]) if form.password.data != form.password_again.data: return render_template( 'register.html', title='Регистрация', form=form, types=types, message=registration_statuses["PASSWORD_MISMATCH"]) registration_request = eval( REGISTRATION["registration_request_structure"]) user = registration_request["user"] user["login"] = form.username.data user["name"] = form.name.data user["surname"] = form.surname.data user["email"] = form.email.data user["address"]["city"] = form.city.data user["address"]["address_line"] = form.address.data user["password_hash"] = hashlib.md5( bytes(form.password.data, encoding='utf-8')).hexdigest() response = requests.post(URL_API + "registration", json=registration_request).json() status = response["authentication_status"] if status == "SUCCESS": return redirect('/login') return render_template("register.html", form=form, types=types, message=registration_statuses[status]) return render_template("register.html", form=form, types=types)
def filter_ids_complete(self, file_: Path): """go over the metadata and get the in/out-bound citation data for each VALID paper. Validity is determined by whether the paper has a proper grobid parse. This method accumulates data into the state, as it won't take so much space. Used by filtering/filter_ids.py. """ file_, valid_pids, min_cite, max_cite, seed = file_ # make sure random.seed(seed) with file_.open("r") as f: # first line in metadata is the header fname = file_.with_suffix("").name next(f) self.results[fname] = [] for line in f: obj = line.strip().split("\t") # has_outbound + has_inbound if (obj[-2] != "[]" and obj[-1] != "[]" and LegacyFilter.validate_grobid(obj)): ob, ib = eval(obj[-2]), eval(obj[-1]) # filter to valid ones vob = [pid for pid in ob if valid_pids.get(pid, False)] vib = [pid for pid in ib if valid_pids.get(pid, False)] if len(vob) >= min_cite and len(vib) >= min_cite: random.shuffle(vob) random.shuffle(vib) self.results[fname].append( (obj[1], vob[:max_cite], vib[:max_cite]))
def test_serialize(self): tests = [ {'body': {}, 'expected': '{}'}, {'body': {'name': 'siri', 'age': 1}, 'expected': '{"name": "siri", "age": 1}'} # TODO: deal with unicode ] for t in tests: resp = mock.Mock() resp.body = t['body'] serialize(dummy, resp, dummy) self.assertDictEqual(eval(resp.body), eval(t['expected']))
def extract(self): # check if not self.extracted: self.extracted = True if self.hasmeta: groups = re.match(self._mdparser_metainfo_regexp, self.text) if groups != None: # Extract infos out info = eval(groups.group(1)) self.info.update(info) # # time-show strategy: # # show the EARLIER time between last_update and t_modify if available # # use GMT-0 when compares, and DO NOT forget to convert the last_update # time_to_show=self.info['t_modify'] # if 'last_update' in self.info: # ftime=time.mktime(time.strptime("+0800 - "+self.info['last_update'], \ # r"%z - %Y/%m/%d %H:%M")) # if ftime<self.info['t_modify']: # time_to_show=ftime # #time zone convert # self.info['time_to_show']=time.strftime(r"%Y/%m/%d %H:%M",\ # dt.utcfromtimestamp(time_to_show).\ # replace(tzinfo=tz.gettz('UTC')).astimezone(tz=tz.gettz('Asia/Shanghai')).\ # timetuple() # ) return self.info
def normalize_these_columns(loader_obj,text): current_values = {} tmp_file = open(text["sortedtoms"] + ".tmp","w") for column in columns: current_values[column] = "" for line in open(text["sortedtoms"]): type, word, id, attrib = line.split('\t') id = id.split() record = Record(type, word, id) record.attrib = eval(attrib) if type == "div1": for column in columns: if column in record.attrib: current_values[column] = record.attrib[column] else: current_values[column] = "" elif type == "div2": for column in columns: if column in record.attrib: current_values[column] = record.attrib[column] elif type == "div3": for column in columns: if column not in record.attrib: record.attrib[column] = current_values[column] print >> tmp_file, record tmp_file.close() os.remove(text["sortedtoms"]) os.rename(text["sortedtoms"] + ".tmp",text["sortedtoms"])
def edit(): secret_key = request.COOKIES.get('secret_key') if secret_key != '2ce67e8fce3cf88e7afa25f08b0332f8': return "Authentication Failure!" data = db.get(request.params['screen_id']) data = eval(data) data["screen_id"] = request.params['screen_id'] data["form_title"] = db.get_cache(request.params['screen_id'])["form_title"] try: data["right_button"]["form_title"] = db.get_cache(data["right_button"]["url"])["form_title"] data["left_button"]["form_title"] = db.get_cache(data["left_button"]["url"])["form_title"] except: pass if "content_url" in data: html_id = data["content_url"].replace("html/", "") html_code = db.get(html_id) data["html_code"] = html_code if 'items' in data.keys(): for i in data['items']: try: form_title = db.get_cache(i["href"])["form_title"] if form_title: i['form_title'] = form_title except: continue template = env.get_template('edit.html') data["screens_list"] = db.get_suggest() # print data return template.render(data)
def prev_next_obj(loader_obj, text, depth=5): object_types = ['doc', 'div1', 'div2', 'div3', 'para', 'sent', 'word'][:depth] record_dict = {} temp_file = text['raw'] + '.tmp' output_file = open(temp_file, 'w') for line in open(text['sortedtoms']): type, word, id, attrib = line.split('\t') id = id.split() record = Record(type, word, id) record.attrib = eval(attrib) if type in record_dict: record_dict[type].attrib['next'] = ' '.join(id) if type in object_types: print >> output_file, record_dict[type] else: del record_dict[type].attrib['next'] del record_dict[type].attrib['prev'] print >> output_file, record_dict[type] record.attrib['prev'] = ' '.join(record_dict[type].id) record_dict[type] = record else: record.attrib['prev'] = '' record_dict[type] = record object_types.reverse() for obj in object_types: record_dict[obj].attrib['next'] = '' print >> output_file, record_dict[obj] output_file.close() os.remove(text['sortedtoms']) tomscommand = "cat %s | egrep \"^doc|^div|^para\" | sort %s > %s" % ( temp_file, loader_obj.sort_by_id, text["sortedtoms"]) os.system(tomscommand) os.remove(temp_file)
def enter_location(GameBoard, Current_XO): while (True): # Enter X,Y location try: print('') enter_XY = eval( input('Enter X,Y value: 0,0 to quit: %s-turn >>> ' % Current_XO)) # print ('enter_location', enter_XY[0], enter_XY[1]) # Swap X,Y -> Y,X Location for easier processing tmp = [enter_XY[0], enter_XY[1] - INDEX[1]] YX_enter = tuple(reversed(tmp)) # Exit due to player quitting if GameBoard[YX_enter[0]][YX_enter[1]] is ' ': print() print('Quitting... Good Bye!!!') exit() # Check if location is valid before placement if find_Current_XO_pair(GameBoard, Current_XO, YX_enter): placement(GameBoard, Current_XO, YX_enter) return else: raise AssertionError # Check invalid X,Y input except (TypeError, ValueError, IndexError, SyntaxError, AssertionError): print('Invalid move. Try again.')
def main(): parser = init_parser(description) args = parser.parse_args() if isinstance(args.target, list): target = args.target[0] log = logging.getLogger('async_write') hdlr = logging.FileHandler(LOGDIR+"/write_to_mongo%s.log" % target) formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%T') hdlr.setFormatter(formatter) log.addHandler(hdlr) log.setLevel(logging.DEBUG) # 把读进来的配置文件转成字典 try: config = eval(args.config.read()) except: config = None log.error(sys.exc_info()) return if config and target in config: rmaster = Redis(**config[target]['rmaster']) rslave = Redis(**config[target]['rslave']) while 1: try: process_async_write(rmaster,rslave,log) except: log.error(traceback.format_exc()) log.error(sys.exc_info()) time.sleep(5*60) else: log.error("conf:%s not exist or async_write_conf.py null" % target)
def generate_frequency(results, q, db, config): """reads through a hitlist. looks up q.frequency_field in each hit, and builds up a list of unique values and their frequencies.""" field = eval(json.loads(q.frequency_field)) ## Override default value of q.end for first batch of results if q.end == 25: q.end = 5000 counts = defaultdict(int) frequency_object = {} try: for hit in results[q.start:q.end]: key = generate_key(hit, field, db) counts[key] += 1 table = {} for key,count in counts.iteritems(): # for each item in the table, we modify the query params to generate a link url. metadata = dict(q.metadata) ## Make a distinct copy for each key in case we modify it below ## Build a label starting with the first value as the main value first_metatada_key, first_metadata_value = key[0] label = first_metadata_value metadata[first_metatada_key] = first_metadata_value.encode('utf-8', 'ignore') append_to_label= [] for metadata_key, metadata_value in key[1:]: if metadata_value == "NULL": metadata[metadata_key] = "NULL" # replace NULL with '[None]', 'N.A.', 'Untitled', etc. else: metadata[metadata_key] = metadata_value.encode('utf-8', 'ignore') # we want to run exact queries on defined values. append_to_label.append(metadata_value) ## Add parentheses to other value, as they are secondary if append_to_label: label = label + ' (' + ', '.join(append_to_label) + ')' ## Quote metadata to force exact matches on metadata for m in metadata: if m not in q.metadata: # skip metadata already in original query: this could be a glob search if metadata[m] and m != "date" and metadata[m] != "NULL": if not metadata[m].startswith('"'): metadata[m] = '"%s"' % metadata[m] # Now build the url from q. url = f.link.make_absolute_query_link(config, q, frequency_field="", start="0", end="0", report=q.report, script='', **metadata) table[label] = {'count': count, 'url': url, 'metadata': metadata} frequency_object['results'] = table frequency_object['more_results'] = True except IndexError: frequency_object['results'] = {} frequency_object['more_results'] = False frequency_object['results_length'] = len(results) frequency_object['query'] = dict([i for i in q]) return frequency_object
def inner_prev_next_obj(loader_obj, text): record_dict = {} temp_file = text['raw'] + '.tmp' output_file = open(temp_file, 'w') for line in open(text['sortedtoms']): type, word, id, attrib = line.split('\t') id = id.split() record = Record(type, word, id) record.attrib = eval(attrib) if type in record_dict: record_dict[type].attrib['next'] = ' '.join(id) if type in types: print >> output_file, record_dict[type] else: del record_dict[type].attrib['next'] del record_dict[type].attrib['prev'] print >> output_file, record_dict[type] record.attrib['prev'] = ' '.join(record_dict[type].id) record_dict[type] = record else: record.attrib['prev'] = '' record_dict[type] = record types.reverse() for obj in types: try: record_dict[obj].attrib['next'] = '' print >> output_file, record_dict[obj] except KeyError: pass output_file.close() os.remove(text['sortedtoms']) type_pattern = "|".join("^%s" % t for t in loader_obj.types) tomscommand = "cat %s | egrep \"%s\" | sort %s > %s" % (temp_file,type_pattern,loader_obj.sort_by_id,text["sortedtoms"]) os.system(tomscommand) os.remove(temp_file)
def prev_next_obj(loader_obj, text, depth=5): object_types = ['doc', 'div1', 'div2', 'div3', 'para', 'sent', 'word'][:depth] record_dict = {} temp_file = text['raw'] + '.tmp' output_file = open(temp_file, 'w') for line in open(text['sortedtoms']): type, word, id, attrib = line.split('\t') id = id.split() record = Record(type, word, id) record.attrib = eval(attrib) if type in record_dict: record_dict[type].attrib['next'] = ' '.join(id) if type in object_types: print >> output_file, record_dict[type] else: del record_dict[type].attrib['next'] del record_dict[type].attrib['prev'] print >> output_file, record_dict[type] record.attrib['prev'] = ' '.join(record_dict[type].id) record_dict[type] = record else: record.attrib['prev'] = '' record_dict[type] = record object_types.reverse() for obj in object_types: record_dict[obj].attrib['next'] = '' print >> output_file, record_dict[obj] output_file.close() os.remove(text['sortedtoms']) tomscommand = "cat %s | egrep \"^doc|^div|^para\" | sort %s > %s" % (temp_file,loader_obj.sort_by_id,text["sortedtoms"]) os.system(tomscommand) os.remove(temp_file)
def extract(self): #check if not self.extracted: self.extracted = True if self.hasmeta: groups = re.match(self._mdparser_metainfo_regexp, self.text) if (groups != None): #Extract infos out info = eval(groups.group(1)) self.info.update(info) # # time-show strategy: # # show the EARLIER time between last_update and t_modify if available # # use GMT-0 when compares, and DO NOT forget to convert the last_update # time_to_show=self.info['t_modify'] # if 'last_update' in self.info: # ftime=time.mktime(time.strptime("+0800 - "+self.info['last_update'], \ # r"%z - %Y/%m/%d %H:%M")) # if ftime<self.info['t_modify']: # time_to_show=ftime # #time zone convert # self.info['time_to_show']=time.strftime(r"%Y/%m/%d %H:%M",\ # dt.utcfromtimestamp(time_to_show).\ # replace(tzinfo=tz.gettz('UTC')).astimezone(tz=tz.gettz('Asia/Shanghai')).\ # timetuple() # ) return self.info
def normalize_these_columns(loader_obj, text): current_values = {} tmp_file = open(text["sortedtoms"] + ".tmp", "w") for column in columns: current_values[column] = "" for line in open(text["sortedtoms"]): type, word, id, attrib = line.split('\t') id = id.split() record = Record(type, word, id) record.attrib = eval(attrib) if type == "div1": for column in columns: if column in record.attrib: current_values[column] = record.attrib[column] else: current_values[column] = "" elif type == "div2": for column in columns: if column in record.attrib: current_values[column] = record.attrib[column] elif type == "div3": for column in columns: if column not in record.attrib: record.attrib[column] = current_values[column] print >> tmp_file, record tmp_file.close() os.remove(text["sortedtoms"]) os.rename(text["sortedtoms"] + ".tmp", text["sortedtoms"])
def get_obj_etag_dict(container): URL = STORAGE_URI+ACCOUNT_NAME+"/" objs = requests.get(URL+container+"?format=JSON", headers={"x-auth-token":TOKEN}) objs_list = eval(objs.text) objs_dict = {} for i in objs_list: objs_dict[i["name"]]=i["hash"] return objs_dict
def get_wyckoffs(sg): wyckoff_strings = eval(wyckoff_df["0"][sg]) wyckoffs = [] for x in wyckoff_strings: wyckoffs.append([]) for y in x: wyckoffs[-1].append(SymmOp.from_xyz_string(y)) return wyckoffs
def login(): form = LoginForm() if form.validate_on_submit(): login_request = eval(config_file["Login"]["login_request_structure"]) login_statuses = eval(config_file["Login"]["login_statuses"]) user = login_request["user"] user["login"] = form.username.data user["password_hash"] = hashlib.md5( bytes(form.password.data, encoding='utf-8')).hexdigest() response = requests.get(URL_API + "login", json=login_request).json() status = response["authentication_status"] if status == "SUCCESS": login_user(convert_user(response), remember=form.remember_me.data) return redirect('/') return render_template('login.html', form=form, message=login_statuses[status]) return render_template('login.html', form=form)
def make_sql_table(self, file_in, table, obj_type='doc'): field_list = ['philo_type', 'philo_name', 'philo_id', 'philo_seq'] depth = object_types.index(obj_type) + 1 ## this is for philo_id slices if table == 'pages': depth = 9 conn = self.dbh c = conn.cursor() if table.endswith('word_counts'): field_list = field_list + ['bytes', '%s_token_count' % obj_type] ## Create table columns = ','.join(field_list) query = 'create table if not exists %s (%s)' % (table, columns) c.execute(query) if table == 'toms': c.execute('create index %s_type_index on %s (philo_type)' % (table,table)) c.execute('create index %s_id_index on %s (philo_id)' % (table,table)) else: c.execute('create index %s_philo_name_index on %s (philo_name)' % (table,table)) c.execute('create index %s_philo_id_index on %s (philo_id)' % (table,table)) conn.commit() sequence = 0 for line in open(file_in): (philo_type,philo_name,id,attrib) = line.split("\t",3) fields = id.split(" ",8) if len(fields) == 9: row = {} if table == "toms": philo_id = " ".join(fields[:7]) elif table == "pages": philo_id = " ".join(fields) elif table.endswith('word_counts'): philo_id = ' '.join(id.split()[:depth]) philo_id = philo_id + ' ' + ' '.join('0' for i in range(7 - depth)) row["philo_type"] = philo_type row["philo_name"] = philo_name row["philo_id"] = philo_id row["philo_seq"] = sequence attrib = eval(attrib) for k in attrib: if k not in field_list: c.execute("ALTER TABLE %s ADD COLUMN %s;" % (table,k)) field_list.append(k) row[k] = attrib[k] row_key = [] row_value = [] for k,v in row.items(): row_key.append(k) row_value.append(v) key_string = "(%s)" % ",".join(x for x in row_key) insert = "INSERT INTO %s %s values (%s);" % (table, key_string,",".join("?" for i in row_value)) c.execute(insert,row_value) sequence += 1 conn.commit()
def tag_words(loader_obj, text): # Set up the treetagger process tt_args = [ tt_path, "-token", "-lemma", "-prob", '-no-unknown', "-threshold", ".01", param_file ] ttout_fh = open(text["raw"] + ".ttout", "w") tt_worker = Popen(tt_args, stdin=PIPE, stdout=ttout_fh) raw_fh = open(text["raw"], "r") line_count = 0 # read through the object file, pass the words to treetagger for line in raw_fh: type, word, id, attrib = line.split('\t') id = id.split() if type == "word": word = word.decode('utf-8', 'ignore').lower().encode('utf-8') # close and re-open the treetagger process to prevent garbage output. if line_count > maxlines: tt_worker.stdin.close() tt_worker.wait() new_ttout_fh = open(text["raw"] + ".ttout", "a") tt_worker = Popen(tt_args, stdin=PIPE, stdout=new_ttout_fh) line_count = 0 print >> tt_worker.stdin, word line_count += 1 # finish tagging tt_worker.stdin.close() tt_worker.wait() # go back through the object file, and add the treetagger results to each word tmp_fh = open(text["raw"] + ".tmp", "w") tag_fh = open(text["raw"] + ".ttout", "r") for line in open(text["raw"], "r"): type, word, id, attrib = line.split('\t') id = id.split() record = Record(type, word, id) record.attrib = eval(attrib) if type == "word": tag_l = tag_fh.readline() next_word, tag = tag_l.split("\t")[0:2] pos, lem, prob = tag.split(" ") if next_word != word.decode('utf-8', 'ignore').lower().encode('utf-8'): print >> sys.stderr, "TREETAGGER ERROR:", next_word, " != ", word, pos, lem return else: record.attrib["pos"] = pos record.attrib["lemma"] = lem print >> tmp_fh, record else: print >> tmp_fh, record os.remove(text["raw"]) os.rename(text["raw"] + ".tmp", text["raw"]) os.remove(text["raw"] + ".ttout")
def fix_pages(loader_obj,text,depth=4): """Unfinished, do not use""" object_types = ['doc', 'div1', 'div2', 'div3', 'para', 'sent', 'word'][:depth] current_page = 0; temp_file = open(text["sortedtoms"] + ".tmp","w") for line in open(text["sortedtoms"]): type, word, id, attrib = line.split('\t') id = id.split() record = Record(type, word, id) record.attrib = eval(attrib)
def test_serialize(self): tests = [{ 'body': {}, 'expected': '{}' }, { 'body': { 'name': 'siri', 'age': 1 }, 'expected': '{"name": "siri", "age": 1}' } # TODO: deal with unicode ] for t in tests: resp = mock.Mock() resp.body = t['body'] serialize(dummy, resp, dummy) self.assertDictEqual(eval(resp.body), eval(t['expected']))
def get(self): ressource_options = default_ressource_options(request, current_app) cursor = current_app.mongo.feeds spec = {} # TODO: check the query if ressource_options['query']: spec = eval(ressource_options['query']) data = [feed for feed in cursor.find(spec=spec)] return dict({'data': data}, **ressource_options)
def word_counts_table(loader_obj, obj_type='doc'): object_types = ['doc', 'div1', 'div2', 'div3', 'para', 'sent', 'word'] depth = object_types.index(obj_type) + 1 ## this is for philo_id slices ## Retrieve column names from toms.db original_fields = ['philo_type', 'philo_name', 'philo_id', 'philo_seq', '%s_token_count' % obj_type] toms_conn = sqlite3.connect(loader_obj.destination + '/toms.db') toms_conn.row_factory = sqlite3.Row toms_c = toms_conn.cursor() toms_c.execute('select * from toms') extra_fields = [i[0] for i in toms_c.description if i[0] not in original_fields] field_list = original_fields + extra_fields + ['bytes'] ## Create table conn = sqlite3.connect(loader_obj.destination + '/%s_word_counts.db' % obj_type) conn.text_factory = str c = conn.cursor() columns = ','.join(field_list) query = 'create table if not exists toms (%s)' % columns c.execute(query) c.execute('create index word_index on toms (philo_name)') c.execute('create index philo_id_index on toms (philo_id)') conn.commit() file_in = '%s/%s_word_counts_sorted' % (loader_obj.workdir, obj_type) sequence = 0 for line in open(file_in): (philo_type,philo_name,id,attrib) = line.split("\t",3) philo_id = ' '.join(id.split()[:depth]) philo_id = philo_id + ' ' + ' '.join('0' for i in range(7 - depth)) row = {} row["philo_type"] = philo_type row["philo_name"] = philo_name row["philo_id"] = philo_id row["philo_seq"] = sequence attrib = eval(attrib) ## Fetch missing fields from toms.db toms_query = 'select %s from toms where philo_id=?' % ','.join(extra_fields) toms_c.execute(toms_query, (philo_id,)) results = [i for i in toms_c.fetchone()] row.update(dict(zip(extra_fields, results))) for k in attrib: row[k] = attrib[k] row_key = [] row_value = [] for k,v in row.items(): row_key.append(k) row_value.append(v) key_string = "(%s)" % ",".join(x for x in row_key) insert = "INSERT INTO toms %s values (%s);" % (key_string,",".join("?" for i in row_value)) c.execute(insert,row_value) sequence += 1 conn.commit()
def run(self): """ Update all tests found in the files given by the `test_suffixes` class attribute. """ # import all the test_suffix experiment files using SourceFileLoader # adapted from: http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path for test_suffix in self.test_suffixes: test_module_path = join( self.tests_directory, 'test_experiment_{}.py'.format(test_suffix)) test_module = SourceFileLoader('loaded_{}'.format(test_suffix), test_module_path).load_module() # skip the module if it tells us that it doesn't want the data for its tests updated if hasattr(test_module, '_AUTO_UPDATE'): if not test_module._AUTO_UPDATE: continue # iterate over all the members and focus on only the experiment functions # but skip over the functions that are decorated with '@raises' since those # functions do not need any test data to be updated. For the rest, try to get # the source since that's what we need to update the test files for member_name, member_object in getmembers(test_module): if isfunction(member_object) and member_name.startswith( 'test_run_experiment'): function = member_object # get the qualified name of the member function member_qualified_name = member_object.__qualname__ # check if it has 'raises' in the qualified name and skip it, if it does if 'raises' in member_qualified_name: continue # otherwise first we check if it's the parameterized function and if so # we can easily get the source from the parameter list if member_name.endswith('parameterized'): for param in function.parameterized_input: source_name = param.args[0] skll = param.kwargs.get('skll', False) self.update_source(source_name, skll=skll) # if it's another function, then we actually inspect the code # to get the source. Note that this should never be a SKLL experiment # since those should always be run parameterized else: function_code_lines = getsourcelines(function) source_line = [ line for line in function_code_lines[0] if re.search(r'source = ', line) ] source_name = eval( source_line[0].strip().split(' = ')[1]) self.update_source(source_name)
def loadmongoconfig(): mdb_transforms = mdb.collection('transforms') mdb_filters = mdb.collection('filters') mdb_selectors = mdb.collection('selectors') mdb_publishers = mdb.collection('publishers') rules = [] for tr in mdb_transforms.find(): if DBG >= 1: logger.debug("transform: %s", tr) if tr['active'] != 'on': logger.info("transform '%s' not active" % tr['transform_name']) continue te = {'name': tr['transform_name']} fil = mdb_filters.find_one({'_id': tr['filter']}) if fil == None: raise SLException("no filter obj %s for transform %s" % (tr['filter'], tr['transform_name'])) te[fil['filter_name']] = eval(fil['filter_string']) sel = mdb_selectors.find_one({'_id': tr['selector']}) if sel == None: raise SLException("no selector obj %s for transform %s" % (tr['selector'], tr['transform_name'])) nsel = eval(sel['selector_string']) if len(nsel) != 0: te[sel['selector_name']] = eval(sel['selector_string']) msel = sel['selector_name'] else: msel = '' pub = mdb_publishers.find_one({'_id': tr['publisher']}) if pub == None: raise SLException("no publisher obj %s for transform %s" % (tr['publisher'], tr['transform_name'])) te[pub['publisher_name']] = eval(pub['publisher_string']) te['rules'] = [[fil['filter_name'], msel, pub['publisher_name']]] rules.append(te) # mdb_nodes = mdb.collection(': return rules
def get_frequency(environ, start_response): status = '200 OK' headers = [('Content-type', 'application/json; charset=UTF-8'), ("Access-Control-Allow-Origin", "*")] start_response(status, headers) config = WebConfig(os.path.abspath(os.path.dirname(__file__)).replace('scripts', '')) db = DB(config.db_path + '/data/') request = WSGIHandler(environ, config) setattr(request, 'frequency_field', simplejson.dumps( eval('"%s"' % request.frequency_field))) results = frequency_results(request, config, sorted=True) yield simplejson.dumps(results)
def normalize_unicode_raw_words(loader_obj, text): tmp_file = open(text["raw"] + ".tmp", "w") for line in open(text["raw"]): rec_type, word, id, attrib = line.split('\t') id = id.split() if rec_type == "word": word = word.decode("utf-8").lower().encode("utf-8") record = Record(rec_type, word, id) record.attrib = eval(attrib) print >> tmp_file, record os.remove(text["raw"]) os.rename(text["raw"] + ".tmp", text["raw"])
def return_point_by_id(self, point_id, retpoint): ##do the select ##consider indexing the table by point id. self.cur_prev.execute("select configpoint, calcsteps, lambda from points where myid = ?", [str(point_id)]) ##save the info r = self.cur_prev.fetchone() # retpoint.append(ast.literal_eval(str(r[0]))) retpoint.append(ast.eval(str(r[0]))) return r[1], r[2]
def normalize_unicode_raw_words(loader_obj, text): tmp_file = open(text["raw"] + ".tmp","w") for line in open(text["raw"]): rec_type, word, id, attrib = line.split('\t') id = id.split() if rec_type == "word": word = word.decode("utf-8").lower().encode("utf-8") record = Record(rec_type, word, id) record.attrib = eval(attrib) print >> tmp_file, record os.remove(text["raw"]) os.rename(text["raw"] + ".tmp",text["raw"])
def latest_version(): '''Returns the latest version number or None if a problem occurs.''' try: # Try reading the repository from GitHub. # Request module is required. from base64 import b64decode from requests import get, codes repo_name = None request = None token = None user = None # Read local information about the repository. try: with data('repo.pydef') as file: repo: dict = eval(file.read()) user = repo['user'] repo_name = repo['name'] except OSError: return print('File repo.pydef not found.') except: return print('Trouble parsing repo dictionary.') # Try to read local token. # Token is necessarry if the repository is private. try: with data('.token') as file: token = file.read().rstrip('\n') except OSError: print('Token not present. Assuming that the repository is public.') # Build url and send request through GitHub API. url = ('https://api.github.com/repos/' f'{user}/{repo_name}/' 'contents/data/version.pydef') if token is None: # Assume that repository is public. No header needed. request = get(url) else: # Token is available. Send it inside the Authorization header. request = get(url, headers={'Authorization': f'token {token}'}) if request.status_code == codes.ok: #pylint: disable=no-member print('Reading latest version from github.') # Before we can read the content, we have to decode it. return int(b64decode(request.json()['content'])) else: return print('Repository or file not found.') except ImportError: return print("Module 'requests' not found.")
def parse_values(input, size): """Parse fuzzy values for the aggregation axis. The input comes from ConfigObj as a list of strings:: >>> print parse_values(["10", "20", "..."], 5) [ 10. 20. 30. 40. 50.] >>> print parse_values(["1", "...", "10"], 5) [ 1. 3.25 5.5 7.75 10. ] >>> print parse_values(["1", "1", "2", "3", "5"], 5) [1 1 2 3 5] Returns a list with the values. """ if len(input) == size and "..." not in input: return np.asarray(map(eval, input)) start, next, stop = input[:] if next == '...': return np.linspace(eval(start), eval(stop), size) elif stop == '...': dx = eval(next)-eval(start) return np.arange(eval(start), eval(start)+size*dx, dx) else: raise Exception('Unable to parse: %s' % input)
def tag_words(loader_obj,text): # Set up the treetagger process tt_args = [tt_path,"-token","-lemma","-prob",'-no-unknown', "-threshold",".01",param_file] ttout_fh = open(text["raw"]+".ttout","w") tt_worker = Popen(tt_args,stdin=PIPE,stdout=ttout_fh) raw_fh = open(text["raw"],"r") line_count = 0 # read through the object file, pass the words to treetagger for line in raw_fh: type, word, id, attrib = line.split('\t') id = id.split() if type == "word": word = word.decode('utf-8', 'ignore').lower().encode('utf-8') # close and re-open the treetagger process to prevent garbage output. if line_count > maxlines: tt_worker.stdin.close() tt_worker.wait() new_ttout_fh = open(text["raw"]+".ttout","a") tt_worker = Popen(tt_args, stdin=PIPE,stdout=new_ttout_fh) line_count = 0 print >> tt_worker.stdin, word line_count += 1 # finish tagging tt_worker.stdin.close() tt_worker.wait() # go back through the object file, and add the treetagger results to each word tmp_fh = open(text["raw"]+".tmp","w") tag_fh = open(text["raw"] + ".ttout","r") for line in open(text["raw"],"r"): type, word, id, attrib = line.split('\t') id = id.split() record = Record(type,word,id) record.attrib = eval(attrib) if type == "word": tag_l = tag_fh.readline() next_word,tag = tag_l.split("\t")[0:2] pos,lem,prob = tag.split(" ") if next_word != word.decode('utf-8', 'ignore').lower().encode('utf-8'): print >> sys.stderr, "TREETAGGER ERROR:",next_word," != ",word,pos,lem return else: record.attrib["pos"] = pos record.attrib["lemma"] = lem print >> tmp_fh, record else: print >> tmp_fh, record os.remove(text["raw"]) os.rename(text["raw"] + ".tmp",text["raw"]) os.remove(text["raw"] + ".ttout")
def return_point_by_id(self, point_id, retpoint): ##do the select ##consider indexing the table by point id. self.cur_prev.execute(\ 'select configpoint, calcsteps, lambda from points where myid = ?', [str(point_id)]) ##save the info r = self.cur_prev.fetchone() #retpoint.append(ast.literal_eval(str(r[0]))) retpoint.append(ast.eval(str(r[0]))) return r[1], r[2]
def loadrules(self): def calldb(collection, func, what=None): q = {'func': func, 'collection': collection} q['what'] = what return self.engine.ask_question("SteamDB", None, q) rules = [] for tr in calldb('transforms', 'find'): if tr['active'] != 'on': self.logger.warn("transform '%s' not active", tr['transform_name']) continue te = {'name': tr['transform_name']} fil = calldb('filters', 'find_one', {'_id': tr['filter']}) if fil == None: raise SLException("no filter obj %s for transform %s" % (tr['filter'], tr['transform_name'])) te[fil['filter_name']] = eval(fil['filter_string']) sel = calldb('selectors', 'find_one', {'_id': tr['selector']}) if sel == None: raise SLException("no selector obj %s for transform %s" % (tr['selector'], tr['transform_name'])) nsel = eval(sel['selector_string']) if len(nsel) != 0: te[sel['selector_name']] = eval(sel['selector_string']) msel = sel['selector_name'] else: msel = '' pub = calldb('publishers', 'find_one', {'_id': tr['publisher']}) if pub == None: raise SLException("no publisher obj %s for transform %s" % (tr['publisher'], tr['transform_name'])) te[pub['publisher_name']] = eval(pub['publisher_string']) te['rules'] = [[fil['filter_name'], msel, pub['publisher_name']]] rules.append(te) return rules
def read_info(): if REPO._INFO_STATUS is None: path, file = fdata_('info.pydef') try: with file: info = eval(file.read()) REPO.USER = info['user'] REPO.NAME = info['repo_name'] REPO.PRIVATE = info['private'] REPO._INFO_STATUS = True except OSError: print(f'File {path} not found.') REPO._INFO_STATUS = False return REPO._INFO_STATUS
def rgb_to_hex(color): """ Converts from rgb to hex Parameters: ----------- color : string Color representation on hex or rgb Example: rgb_to_hex('rgb(23,25,24)') """ rgb=eval(color.replace('rgb','')) return '#' + ''.join(['{0:02x}'.format(x) for x in rgb])
def checkForCoreFiles(self): try: self.coreSearch = self.run( '%s ls -lrt %s' % (self.setsudo, self.coresearch), sudo=self.sudo) if len(self.coreSearch) == 0: self.coreSearch = False elif self.coreSearch.split()[1] == "cannot" and self.coreSearch.split()[2] == "access": self.coreSearch = False else: self.coreSearch = eval(self.coreSearch) except BaseException: #print "problem in tracing the core files.Please check manually for the existence of core file." self.coreSearch = False
def process_corpus(self): lines = [eval(line.strip()) for line in open("tweetlabels.txt", "r")] stop = stopwords.words("english") stop.remove("no") corpus = [x[0] for x in lines] labels = [x[1] for x in lines] pr = Processor.Processor(stop) normalized_corpus, lines = pr.process(corpus) del lines tfidf = Tfidf(ngram_range=(1,2), binary=True, tokenizer=tweet_tokenizer) X = tfidf.fit_transform(normalized_corpus) self.X = X self.labels = labels
def collocation_fetcher(environ,start_response): status = '200 OK' headers = [('Content-type', 'application/json; charset=UTF-8'),("Access-Control-Allow-Origin","*")] start_response(status,headers) environ["SCRIPT_FILENAME"] = environ["SCRIPT_FILENAME"].replace('scripts/collocation_fetcher.py', '') cgi = urlparse.parse_qs(environ["QUERY_STRING"],keep_blank_values=True) full_report = eval(cgi.get('full_report',['True'])[0]) db, path_components, q = parse_cgi(environ) hits = db.query(q["q"],q["method"],q["arg"],**q["metadata"]) if full_report: all_colloc, left_colloc, right_colloc = r.fetch_collocation(hits, environ["SCRIPT_FILENAME"], q, db) yield dumps([all_colloc, left_colloc, right_colloc]) else: results = r.fetch_collocation(hits, environ["SCRIPT_FILENAME"], q, db, full_report=False) yield dumps(results)
def init_from_str(self): r = r"([a-zA-Z]*)=([^,)]*)" opt_str = self.param.lower() kwargs = dict([(i, eval(j)) for (i, j) in re.findall(r, opt_str)]) if "sgd" in opt_str: optimizer = SGD(**kwargs) elif "adagrad" in opt_str: optimizer = AdaGrad(**kwargs) elif "rmsprop" in opt_str: optimizer = RMSProp(**kwargs) elif "adam" in opt_str: optimizer = Adam(**kwargs) else: raise NotImplementedError("{}".format(opt_str)) return optimizer
def main(): """ Indexes MongoDB """ parser = init_parser(description) args = parser.parse_args() # 把读进来的配置文件转成字典 config = eval(args.config.read()) log.debug("\n") the_mongo = MONGODB if isinstance(the_mongo, dict): process_index('mongodb', the_mongo, config) process_index('log_mongodb', the_mongo, config)
def run(self): """ Update all tests found in the files given by the `test_suffixes` class attribute. """ # import all the test_suffix experiment files using SourceFileLoader # adapted from: http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path for test_suffix in self.test_suffixes: test_module_path = join(self.tests_directory, 'test_experiment_{}.py'.format(test_suffix)) test_module = SourceFileLoader('loaded_{}'.format(test_suffix), test_module_path).load_module() # skip the module if it tells us that it doesn't want the data for its tests updated if hasattr(test_module, '_AUTO_UPDATE'): if not test_module._AUTO_UPDATE: continue # iterate over all the members and focus on only the experiment functions # but skip over the functions that are decorated with '@raises' since those # functions do not need any test data to be updated. For the rest, try to get # the source since that's what we need to update the test files for member_name, member_object in getmembers(test_module): if isfunction(member_object) and member_name.startswith('test_run_experiment'): function = member_object # get the qualified name of the member function member_qualified_name = member_object.__qualname__ # check if it has 'raises' in the qualified name and skip it, if it does if 'raises' in member_qualified_name: continue # otherwise first we check if it's the parameterized function and if so # we can easily get the source from the parameter list if member_name.endswith('parameterized'): for param in function.parameterized_input: source_name = param.args[0] skll = param.kwargs.get('skll', False) self.update_source(source_name, skll=skll) # if it's another function, then we actually inspect the code # to get the source. Note that this should never be a SKLL experiment # since those should always be run parameterized else: function_code_lines = getsourcelines(function) source_line = [line for line in function_code_lines[0] if re.search(r'source = ', line)] source_name = eval(source_line[0].strip().split(' = ')[1]) self.update_source(source_name)
def inner_make_object_ancestors(loader_obj, text): temp_file = text['words'] + '.tmp' output_file = open(temp_file, 'w') for line in open(text['words']): type, word, id, attrib = line.split('\t') id = id.split() record = Record(type, word, id) record.attrib = eval(attrib) for type in types: zeros_to_add = ['0' for i in range(7 - type_depth[type])] philo_id = id[:type_depth[type]] + zeros_to_add record.attrib[type + '_ancestor'] = ' '.join(philo_id) print >> output_file, record output_file.close() os.remove(text['words']) os.rename(temp_file, text['words'])
def init_from_str(self): r = r"([a-zA-Z]*)=([^,)]*)" sch_str = self.param.lower() kwargs = dict([(i, eval(j)) for (i, j) in re.findall(r, sch_str)]) if "constant" in sch_str: scheduler = ConstantScheduler(**kwargs) elif "exponential" in sch_str: scheduler = ExponentialScheduler(**kwargs) elif "noam" in sch_str: scheduler = NoamScheduler(**kwargs) elif "king" in sch_str: scheduler = KingScheduler(**kwargs) else: raise NotImplementedError("{}".format(sch_str)) return scheduler
def game_loop(board, piece): print() print_board(board) while (True): try: move = eval(input('Place %s where? ' % piece)) move = tuple(reversed(move)) # x,y -> y,x (easier to use) if is_valid_move(board, piece, move): place_piece(board, piece, move) return else: raise AssertionError except (TypeError, ValueError, IndexError, SyntaxError, AssertionError): # ------------------bad input------------------ ---bad move--- print('Invalid move. Try again.')
def make_sql_table(self, table, file_in, indices=[], depth=7): conn = self.dbh c = conn.cursor() columns = 'philo_type,philo_name,philo_id,philo_seq' query = 'create table if not exists %s (%s)' % (table, columns) c.execute(query) print "%s table in toms.db database file..." % table, sequence = 0 for line in open(file_in): philo_type, philo_name, id, attrib = line.split("\t",3) fields = id.split(" ",8) if len(fields) == 9: row = {} philo_id = " ".join(fields[:depth]) row["philo_type"] = philo_type row["philo_name"] = philo_name row["philo_id"] = philo_id row["philo_seq"] = sequence row.update(eval(attrib)) columns = "(%s)" % ",".join([i for i in row]) insert = "INSERT INTO %s %s values (%s);" % (table, columns,",".join(["?" for i in row])) values = [v for k,v in row.items()] try: c.execute(insert,values) except sqlite3.OperationalError: c.execute("PRAGMA table_info(%s)" % table) column_list = [i[1] for i in c.fetchall()] for column in row: if column not in column_list: c.execute("ALTER TABLE %s ADD COLUMN %s;" % (table,column)) c.execute(insert,values) sequence += 1 conn.commit() for index in indices: try: c.execute('create index if not exists %s_%s_index on %s (%s)' % (table,index,table,index)) except sqlite3.OperationalError: pass conn.commit() print 'done.' if self.clean: os.system('rm %s' % file_in)
def restore(self, obj): try: restore_method = [ m for m in self.methods if isinstance(obj, m.type) ] if not restore_method: restore_method = [ m for m in self.methods if obj.startswith(m.name + '-->') ] if not restore_method: restore_method = [ t for t in ntypes if obj.startswith(str(t)[8:-2] + '-->') ] if restore_method: return eval(obj[len(str(restore_method[0])) - 7:]) return restore_method[0].defreezer(obj, self) except: return obj
def main(): """ Indexes MongoDB """ parser = init_parser(description) args = parser.parse_args() # 把读进来的配置文件转成字典 config = eval(args.config.read()) if isinstance(args.target, list): for target in args.target: log.debug("切换环境: %s" % target) log.debug("\n") the_mongo = MONGODB[target] if isinstance(the_mongo, dict): process_index('mongodb', the_mongo, config) process_index('log_mongodb', the_mongo, config)
def smash_these_unicode_columns(loader_obj, text): tmp_file = open(text["sortedtoms"] + ".tmp","w") for line in open(text["sortedtoms"]): type, word, id, attrib = line.split('\t') id = id.split() record = Record(type, word, id) record.attrib = eval(attrib) for column in columns: if column in record.attrib: # print >> sys.stderr, repr(record.attrib) col = record.attrib[column].decode("utf-8") col = col.lower() smashed_col = [c for c in unicodedata.normalize("NFKD",col) if not unicodedata.combining(c)] record.attrib[column + "_norm"] = ''.join(smashed_col).encode("utf-8") #record.attrib[column + "_norm"] = ''.join([c.encode("utf-8") for c in unicodedata.normalize('NFKD',record.attrib[column].decode("utf-8").lower()) if not unicodedata.combining(c)]) print >> tmp_file, record tmp_file.close() os.remove(text["sortedtoms"]) os.rename(text["sortedtoms"] + ".tmp",text["sortedtoms"])
def make_word_counts(loader_obj, text, depth=4): object_types = ['doc', 'div1', 'div2', 'div3', 'para', 'sent', 'word'] counts = [0 for i in range(depth)] temp_file = text['raw'] + '.tmp' output_file = open(temp_file, 'w') for line in open(text['raw']): type, word, id, attrib = line.split('\t') id = id.split() record = Record(type, word, id) record.attrib = eval(attrib) for d, count in enumerate(counts): if type == 'word': counts[d] += 1 elif type == object_types[d]: record.attrib['word_count'] = counts[d] counts[d] = 0 print >> output_file, record output_file.close() os.remove(text['raw']) os.rename(temp_file, text['raw'])
def normalize(self, dataset, datatype="normal"): print("Normalizing..") if datatype == "normal": if len( dataset[0] ) > self.trainingsize: # check if new data is larger than training sample(it will be just one set, as a list) self.addtotraining(len(dataset[0])) self.train("retrain") newdataset = [] for data in dataset: #if type(data) == str: data = eval(str(data)) newdataset.append(data) if len(data) > self.datasize: self.datasize = len(data) dataset = newdataset trainingdata = Settings().getdata("MACHINELEARNING", "trainingfeatures") for data in dataset: print("comparing datalength!") while len(data) != self.datasize: if len(data) < self.datasize: print("new data is smaller!") data.append(9999) else: print("new data is larger!") for tdata in trainingdata: while len(data) != len(tdata): tdata.append(9999) if datatype == "training": self.trainingsize = self.datasize # set training sample size Settings().setdata( {"MACHINELEARNING": { "trainingsize": self.trainingsize }}) self.trainingfeaturesaslist = dataset Settings().setdata( {"MACHINELEARNING": { "normalizeddataset": dataset }}) return dataset
def read_footer(ifolder=None,tfile=None,ext='all'): from sq_tools import read_footer from ast import literal_eval as eval if ext=='all': ext=['nsp','nsx','nsz','xci','xcz'] else: if isinstance(ext, list): ext=ext else: try: ext=eval(ext) except:pass ext=ext.split(',') if ifolder!=None: files=listmanager.folder_to_list(ifolder,ext) elif tfile!=None: files=read_lines_to_list(tfile,all=True) else: return False for filepath in files: read_footer(filepath)
def word_frequencies_per_obj(loader_obj, text, depth=1): object_types = ['doc', 'div1', 'div2', 'div3', 'para', 'sent', 'word'][:depth] files_path = loader_obj.destination + '/WORK/' try: os.mkdir(files_path) except OSError: ## Path was already created pass for d, obj in enumerate(object_types): file = text['name'] + '.%s.sorted' % obj output = open(files_path + file, 'w') d = d + 1 old_philo_id = [] records = {} for line in open(text['words']): type, word, id, attrib = line.split('\t') attrib = eval(attrib) philo_id = id.split() record = Record(type, word, philo_id) count_key = obj + '_token_count' byte = attrib['byte_start'] del attrib['byte_start'] record.attrib = {count_key: attrib[count_key]} if philo_id[:d] != old_philo_id[:d]: if records: for w in records: print >> output, records[w] records = {} if word not in records: record.attrib['bytes'] = [] record.attrib['bytes'] = str(byte) records[word] = record else: records[word].attrib['bytes'] += ' ' + str(byte) old_philo_id = philo_id for w in records: print >> output, records[w] output.close()