def to_rgba(color,alpha):
"""
Converts from hex|rgb to rgba
Parameters:
-----------
color : string
Color representation on hex or rgb
alpha : float
Value from 0 to 1.0 that represents the
alpha value.
Example:
to_rgba('#E1E5ED',0.6)
to_rgba('#f03',0.7)
to_rgba('rgb(23,23,23)',.5)
"""
color=color.lower()
if 'rgba' in color:
cl=list(eval(color.replace('rgba','')))
cl[3]=alpha
return 'rgba'+str(tuple(cl))
elif 'rgb' in color:
r,g,b=eval(color.replace('rgb',''))
return 'rgba'+str((r,g,b,alpha))
else:
return to_rgba(hex_to_rgb(color),alpha)
def run(ast, args=['<progname>'], check_types=False):
ast = rewrite(ast, replace_nodes)
log.final_ast("the final AST is:\n", ast)
frame = Frame()
ast.eval(frame)
log.topframe("the top frame is\n", frame)
if 'main' not in frame:
print("no main function defined, exiting")
return 0
# type inference
if check_types:
with frame as newframe:
newframe['argc'] = Int(len(args))
newframe['argv'] = Array(map(Str, args))
main = newframe['main']
main.infer_type(newframe)
assert main.type.ret == Int, \
"main() should return Int but got %s" % main.type.ret
with frame as newframe:
newframe['argc'] = Int(len(args))
newframe['argv'] = Array(map(Str, args))
r = newframe['main'].Call(newframe)
if isinstance(r, Int):
return r.to_int()
else:
return 0
def __init__(self):
_pos = file("pos", "r")
_neg = file("neg", "r")
self.pos = eval(_pos.readline().strip())
self.neg = eval(_neg.readline().strip())
_pos.close()
_neg.close()
def add_signed_footer(message=None,ifolder=None,tfile=None,ext='all',rewrite=False):
from sq_tools import add_signed_footer as sign
from ast import literal_eval as eval
if ext=='all':
ext=['nsp','nsx','nsz','xci','xcz']
else:
if isinstance(ext, list):
ext=ext
else:
try:
ext=eval(ext)
except:pass
ext=ext.split(',')
if ifolder!=None:
try:
ifolder=eval(ifolder)
except:pass
files=listmanager.folder_to_list(ifolder,ext)
elif tfile!=None:
try:
tfile=eval(tfile)
except:pass
files=read_lines_to_list(tfile,all=True)
else:
return False
for filepath in files:
try:
sign(filepath,message,rewrite)
except:
print('Error in '+filepath)
def register():
form = RegisterForm()
types = [type(field).__name__ for field in form]
if form.validate_on_submit():
registration_statuses = eval(REGISTRATION["registration_statuses"])
if form.password.data != form.password_again.data:
return render_template(
'register.html',
title='Регистрация',
form=form,
types=types,
message=registration_statuses["PASSWORD_MISMATCH"])
registration_request = eval(
REGISTRATION["registration_request_structure"])
user = registration_request["user"]
user["login"] = form.username.data
user["name"] = form.name.data
user["surname"] = form.surname.data
user["email"] = form.email.data
user["address"]["city"] = form.city.data
user["address"]["address_line"] = form.address.data
user["password_hash"] = hashlib.md5(
bytes(form.password.data, encoding='utf-8')).hexdigest()
response = requests.post(URL_API + "registration",
json=registration_request).json()
status = response["authentication_status"]
if status == "SUCCESS":
return redirect('/login')
return render_template("register.html",
form=form,
types=types,
message=registration_statuses[status])
return render_template("register.html", form=form, types=types)
def run(ast, args=['<progname>'], check_types=False):
ast = rewrite(ast, replace_nodes)
log.final_ast("the final AST is:\n", ast)
frame = Frame()
ast.eval(frame)
log.topframe("the top frame is\n", frame)
if 'main' not in frame:
print("no main function defined, exiting")
return 0
# type inference
if check_types:
with frame as newframe:
newframe['argc'] = Int(len(args))
newframe['argv'] = Array(map(Str, args))
main = newframe['main']
main.infer_type(newframe)
assert main.type.ret == Int, \
"main() should return Int but got %s" % main.type.ret
with frame as newframe:
newframe['argc'] = Int(len(args))
newframe['argv'] = Array(map(Str, args))
r = newframe['main'].Call(newframe)
if isinstance(r, Int):
return r.to_int()
else:
return 0
def filter_ids_complete(self, file_: Path):
"""go over the metadata and get the in/out-bound citation data for each VALID
paper. Validity is determined by whether the paper has a proper grobid parse.
This method accumulates data into the state, as it won't take so much space.
Used by filtering/filter_ids.py.
"""
file_, valid_pids, min_cite, max_cite, seed = file_
# make sure
random.seed(seed)
with file_.open("r") as f:
# first line in metadata is the header
fname = file_.with_suffix("").name
next(f)
self.results[fname] = []
for line in f:
obj = line.strip().split("\t")
# has_outbound + has_inbound
if (obj[-2] != "[]" and obj[-1] != "[]"
and LegacyFilter.validate_grobid(obj)):
ob, ib = eval(obj[-2]), eval(obj[-1])
# filter to valid ones
vob = [pid for pid in ob if valid_pids.get(pid, False)]
vib = [pid for pid in ib if valid_pids.get(pid, False)]
if len(vob) >= min_cite and len(vib) >= min_cite:
random.shuffle(vob)
random.shuffle(vib)
self.results[fname].append(
(obj[1], vob[:max_cite], vib[:max_cite]))
def test_serialize(self):
tests = [
{'body': {}, 'expected': '{}'},
{'body': {'name': 'siri', 'age': 1}, 'expected': '{"name": "siri", "age": 1}'}
# TODO: deal with unicode
]
for t in tests:
resp = mock.Mock()
resp.body = t['body']
serialize(dummy, resp, dummy)
self.assertDictEqual(eval(resp.body), eval(t['expected']))
def extract(self):
# check
if not self.extracted:
self.extracted = True
if self.hasmeta:
groups = re.match(self._mdparser_metainfo_regexp, self.text)
if groups != None:
# Extract infos out
info = eval(groups.group(1))
self.info.update(info)
# # time-show strategy:
# # show the EARLIER time between last_update and t_modify if available
# # use GMT-0 when compares, and DO NOT forget to convert the last_update
# time_to_show=self.info['t_modify']
# if 'last_update' in self.info:
# ftime=time.mktime(time.strptime("+0800 - "+self.info['last_update'], \
# r"%z - %Y/%m/%d %H:%M"))
# if ftime<self.info['t_modify']:
# time_to_show=ftime
# #time zone convert
# self.info['time_to_show']=time.strftime(r"%Y/%m/%d %H:%M",\
# dt.utcfromtimestamp(time_to_show).\
# replace(tzinfo=tz.gettz('UTC')).astimezone(tz=tz.gettz('Asia/Shanghai')).\
# timetuple()
# )
return self.info
def normalize_these_columns(loader_obj,text):
current_values = {}
tmp_file = open(text["sortedtoms"] + ".tmp","w")
for column in columns:
current_values[column] = ""
for line in open(text["sortedtoms"]):
type, word, id, attrib = line.split('\t')
id = id.split()
record = Record(type, word, id)
record.attrib = eval(attrib)
if type == "div1":
for column in columns:
if column in record.attrib:
current_values[column] = record.attrib[column]
else:
current_values[column] = ""
elif type == "div2":
for column in columns:
if column in record.attrib:
current_values[column] = record.attrib[column]
elif type == "div3":
for column in columns:
if column not in record.attrib:
record.attrib[column] = current_values[column]
print >> tmp_file, record
tmp_file.close()
os.remove(text["sortedtoms"])
os.rename(text["sortedtoms"] + ".tmp",text["sortedtoms"])
def edit():
secret_key = request.COOKIES.get('secret_key')
if secret_key != '2ce67e8fce3cf88e7afa25f08b0332f8':
return "Authentication Failure!"
data = db.get(request.params['screen_id'])
data = eval(data)
data["screen_id"] = request.params['screen_id']
data["form_title"] = db.get_cache(request.params['screen_id'])["form_title"]
try:
data["right_button"]["form_title"] = db.get_cache(data["right_button"]["url"])["form_title"]
data["left_button"]["form_title"] = db.get_cache(data["left_button"]["url"])["form_title"]
except:
pass
if "content_url" in data:
html_id = data["content_url"].replace("html/", "")
html_code = db.get(html_id)
data["html_code"] = html_code
if 'items' in data.keys():
for i in data['items']:
try:
form_title = db.get_cache(i["href"])["form_title"]
if form_title:
i['form_title'] = form_title
except:
continue
template = env.get_template('edit.html')
data["screens_list"] = db.get_suggest()
# print data
return template.render(data)
def prev_next_obj(loader_obj, text, depth=5):
object_types = ['doc', 'div1', 'div2', 'div3', 'para', 'sent',
'word'][:depth]
record_dict = {}
temp_file = text['raw'] + '.tmp'
output_file = open(temp_file, 'w')
for line in open(text['sortedtoms']):
type, word, id, attrib = line.split('\t')
id = id.split()
record = Record(type, word, id)
record.attrib = eval(attrib)
if type in record_dict:
record_dict[type].attrib['next'] = ' '.join(id)
if type in object_types:
print >> output_file, record_dict[type]
else:
del record_dict[type].attrib['next']
del record_dict[type].attrib['prev']
print >> output_file, record_dict[type]
record.attrib['prev'] = ' '.join(record_dict[type].id)
record_dict[type] = record
else:
record.attrib['prev'] = ''
record_dict[type] = record
object_types.reverse()
for obj in object_types:
record_dict[obj].attrib['next'] = ''
print >> output_file, record_dict[obj]
output_file.close()
os.remove(text['sortedtoms'])
tomscommand = "cat %s | egrep \"^doc|^div|^para\" | sort %s > %s" % (
temp_file, loader_obj.sort_by_id, text["sortedtoms"])
os.system(tomscommand)
os.remove(temp_file)
def enter_location(GameBoard, Current_XO):
while (True):
# Enter X,Y location
try:
print('')
enter_XY = eval(
input('Enter X,Y value: 0,0 to quit: %s-turn >>> ' %
Current_XO))
# print ('enter_location', enter_XY[0], enter_XY[1])
# Swap X,Y -> Y,X Location for easier processing
tmp = [enter_XY[0], enter_XY[1] - INDEX[1]]
YX_enter = tuple(reversed(tmp))
# Exit due to player quitting
if GameBoard[YX_enter[0]][YX_enter[1]] is ' ':
print()
print('Quitting... Good Bye!!!')
exit()
# Check if location is valid before placement
if find_Current_XO_pair(GameBoard, Current_XO, YX_enter):
placement(GameBoard, Current_XO, YX_enter)
return
else:
raise AssertionError
# Check invalid X,Y input
except (TypeError, ValueError, IndexError, SyntaxError,
AssertionError):
print('Invalid move. Try again.')
def main():
parser = init_parser(description)
args = parser.parse_args()
if isinstance(args.target, list):
target = args.target[0]
log = logging.getLogger('async_write')
hdlr = logging.FileHandler(LOGDIR+"/write_to_mongo%s.log" % target)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%T')
hdlr.setFormatter(formatter)
log.addHandler(hdlr)
log.setLevel(logging.DEBUG)
# 把读进来的配置文件转成字典
try:
config = eval(args.config.read())
except:
config = None
log.error(sys.exc_info())
return
if config and target in config:
rmaster = Redis(**config[target]['rmaster'])
rslave = Redis(**config[target]['rslave'])
while 1:
try:
process_async_write(rmaster,rslave,log)
except:
log.error(traceback.format_exc())
log.error(sys.exc_info())
time.sleep(5*60)
else:
log.error("conf:%s not exist or async_write_conf.py null" % target)
def generate_frequency(results, q, db, config):
"""reads through a hitlist. looks up q.frequency_field in each hit, and builds up a list of
unique values and their frequencies."""
field = eval(json.loads(q.frequency_field))
## Override default value of q.end for first batch of results
if q.end == 25:
q.end = 5000
counts = defaultdict(int)
frequency_object = {}
try:
for hit in results[q.start:q.end]:
key = generate_key(hit, field, db)
counts[key] += 1
table = {}
for key,count in counts.iteritems():
# for each item in the table, we modify the query params to generate a link url.
metadata = dict(q.metadata) ## Make a distinct copy for each key in case we modify it below
## Build a label starting with the first value as the main value
first_metatada_key, first_metadata_value = key[0]
label = first_metadata_value
metadata[first_metatada_key] = first_metadata_value.encode('utf-8', 'ignore')
append_to_label= []
for metadata_key, metadata_value in key[1:]:
if metadata_value == "NULL":
metadata[metadata_key] = "NULL" # replace NULL with '[None]', 'N.A.', 'Untitled', etc.
else:
metadata[metadata_key] = metadata_value.encode('utf-8', 'ignore') # we want to run exact queries on defined values.
append_to_label.append(metadata_value)
## Add parentheses to other value, as they are secondary
if append_to_label:
label = label + ' (' + ', '.join(append_to_label) + ')'
## Quote metadata to force exact matches on metadata
for m in metadata:
if m not in q.metadata: # skip metadata already in original query: this could be a glob search
if metadata[m] and m != "date" and metadata[m] != "NULL":
if not metadata[m].startswith('"'):
metadata[m] = '"%s"' % metadata[m]
# Now build the url from q.
url = f.link.make_absolute_query_link(config, q, frequency_field="", start="0", end="0", report=q.report, script='', **metadata)
table[label] = {'count': count, 'url': url, 'metadata': metadata}
frequency_object['results'] = table
frequency_object['more_results'] = True
except IndexError:
frequency_object['results'] = {}
frequency_object['more_results'] = False
frequency_object['results_length'] = len(results)
frequency_object['query'] = dict([i for i in q])
return frequency_object
def inner_prev_next_obj(loader_obj, text):
record_dict = {}
temp_file = text['raw'] + '.tmp'
output_file = open(temp_file, 'w')
for line in open(text['sortedtoms']):
type, word, id, attrib = line.split('\t')
id = id.split()
record = Record(type, word, id)
record.attrib = eval(attrib)
if type in record_dict:
record_dict[type].attrib['next'] = ' '.join(id)
if type in types:
print >> output_file, record_dict[type]
else:
del record_dict[type].attrib['next']
del record_dict[type].attrib['prev']
print >> output_file, record_dict[type]
record.attrib['prev'] = ' '.join(record_dict[type].id)
record_dict[type] = record
else:
record.attrib['prev'] = ''
record_dict[type] = record
types.reverse()
for obj in types:
try:
record_dict[obj].attrib['next'] = ''
print >> output_file, record_dict[obj]
except KeyError:
pass
output_file.close()
os.remove(text['sortedtoms'])
type_pattern = "|".join("^%s" % t for t in loader_obj.types)
tomscommand = "cat %s | egrep \"%s\" | sort %s > %s" % (temp_file,type_pattern,loader_obj.sort_by_id,text["sortedtoms"])
os.system(tomscommand)
os.remove(temp_file)
def prev_next_obj(loader_obj, text, depth=5):
object_types = ['doc', 'div1', 'div2', 'div3', 'para', 'sent', 'word'][:depth]
record_dict = {}
temp_file = text['raw'] + '.tmp'
output_file = open(temp_file, 'w')
for line in open(text['sortedtoms']):
type, word, id, attrib = line.split('\t')
id = id.split()
record = Record(type, word, id)
record.attrib = eval(attrib)
if type in record_dict:
record_dict[type].attrib['next'] = ' '.join(id)
if type in object_types:
print >> output_file, record_dict[type]
else:
del record_dict[type].attrib['next']
del record_dict[type].attrib['prev']
print >> output_file, record_dict[type]
record.attrib['prev'] = ' '.join(record_dict[type].id)
record_dict[type] = record
else:
record.attrib['prev'] = ''
record_dict[type] = record
object_types.reverse()
for obj in object_types:
record_dict[obj].attrib['next'] = ''
print >> output_file, record_dict[obj]
output_file.close()
os.remove(text['sortedtoms'])
tomscommand = "cat %s | egrep \"^doc|^div|^para\" | sort %s > %s" % (temp_file,loader_obj.sort_by_id,text["sortedtoms"])
os.system(tomscommand)
os.remove(temp_file)
def extract(self):
#check
if not self.extracted:
self.extracted = True
if self.hasmeta:
groups = re.match(self._mdparser_metainfo_regexp, self.text)
if (groups != None):
#Extract infos out
info = eval(groups.group(1))
self.info.update(info)
# # time-show strategy:
# # show the EARLIER time between last_update and t_modify if available
# # use GMT-0 when compares, and DO NOT forget to convert the last_update
# time_to_show=self.info['t_modify']
# if 'last_update' in self.info:
# ftime=time.mktime(time.strptime("+0800 - "+self.info['last_update'], \
# r"%z - %Y/%m/%d %H:%M"))
# if ftime<self.info['t_modify']:
# time_to_show=ftime
# #time zone convert
# self.info['time_to_show']=time.strftime(r"%Y/%m/%d %H:%M",\
# dt.utcfromtimestamp(time_to_show).\
# replace(tzinfo=tz.gettz('UTC')).astimezone(tz=tz.gettz('Asia/Shanghai')).\
# timetuple()
# )
return self.info
def normalize_these_columns(loader_obj, text):
current_values = {}
tmp_file = open(text["sortedtoms"] + ".tmp", "w")
for column in columns:
current_values[column] = ""
for line in open(text["sortedtoms"]):
type, word, id, attrib = line.split('\t')
id = id.split()
record = Record(type, word, id)
record.attrib = eval(attrib)
if type == "div1":
for column in columns:
if column in record.attrib:
current_values[column] = record.attrib[column]
else:
current_values[column] = ""
elif type == "div2":
for column in columns:
if column in record.attrib:
current_values[column] = record.attrib[column]
elif type == "div3":
for column in columns:
if column not in record.attrib:
record.attrib[column] = current_values[column]
print >> tmp_file, record
tmp_file.close()
os.remove(text["sortedtoms"])
os.rename(text["sortedtoms"] + ".tmp", text["sortedtoms"])
def get_obj_etag_dict(container):
URL = STORAGE_URI+ACCOUNT_NAME+"/"
objs = requests.get(URL+container+"?format=JSON", headers={"x-auth-token":TOKEN})
objs_list = eval(objs.text)
objs_dict = {}
for i in objs_list:
objs_dict[i["name"]]=i["hash"]
return objs_dict
def get_wyckoffs(sg):
wyckoff_strings = eval(wyckoff_df["0"][sg])
wyckoffs = []
for x in wyckoff_strings:
wyckoffs.append([])
for y in x:
wyckoffs[-1].append(SymmOp.from_xyz_string(y))
return wyckoffs
def login():
form = LoginForm()
if form.validate_on_submit():
login_request = eval(config_file["Login"]["login_request_structure"])
login_statuses = eval(config_file["Login"]["login_statuses"])
user = login_request["user"]
user["login"] = form.username.data
user["password_hash"] = hashlib.md5(
bytes(form.password.data, encoding='utf-8')).hexdigest()
response = requests.get(URL_API + "login", json=login_request).json()
status = response["authentication_status"]
if status == "SUCCESS":
login_user(convert_user(response), remember=form.remember_me.data)
return redirect('/')
return render_template('login.html',
form=form,
message=login_statuses[status])
return render_template('login.html', form=form)
def make_sql_table(self, file_in, table, obj_type='doc'):
field_list = ['philo_type', 'philo_name', 'philo_id', 'philo_seq']
depth = object_types.index(obj_type) + 1 ## this is for philo_id slices
if table == 'pages':
depth = 9
conn = self.dbh
c = conn.cursor()
if table.endswith('word_counts'):
field_list = field_list + ['bytes', '%s_token_count' % obj_type]
## Create table
columns = ','.join(field_list)
query = 'create table if not exists %s (%s)' % (table, columns)
c.execute(query)
if table == 'toms':
c.execute('create index %s_type_index on %s (philo_type)' % (table,table))
c.execute('create index %s_id_index on %s (philo_id)' % (table,table))
else:
c.execute('create index %s_philo_name_index on %s (philo_name)' % (table,table))
c.execute('create index %s_philo_id_index on %s (philo_id)' % (table,table))
conn.commit()
sequence = 0
for line in open(file_in):
(philo_type,philo_name,id,attrib) = line.split("\t",3)
fields = id.split(" ",8)
if len(fields) == 9:
row = {}
if table == "toms":
philo_id = " ".join(fields[:7])
elif table == "pages":
philo_id = " ".join(fields)
elif table.endswith('word_counts'):
philo_id = ' '.join(id.split()[:depth])
philo_id = philo_id + ' ' + ' '.join('0' for i in range(7 - depth))
row["philo_type"] = philo_type
row["philo_name"] = philo_name
row["philo_id"] = philo_id
row["philo_seq"] = sequence
attrib = eval(attrib)
for k in attrib:
if k not in field_list:
c.execute("ALTER TABLE %s ADD COLUMN %s;" % (table,k))
field_list.append(k)
row[k] = attrib[k]
row_key = []
row_value = []
for k,v in row.items():
row_key.append(k)
row_value.append(v)
key_string = "(%s)" % ",".join(x for x in row_key)
insert = "INSERT INTO %s %s values (%s);" % (table, key_string,",".join("?" for i in row_value))
c.execute(insert,row_value)
sequence += 1
conn.commit()
def tag_words(loader_obj, text):
# Set up the treetagger process
tt_args = [
tt_path, "-token", "-lemma", "-prob", '-no-unknown', "-threshold",
".01", param_file
]
ttout_fh = open(text["raw"] + ".ttout", "w")
tt_worker = Popen(tt_args, stdin=PIPE, stdout=ttout_fh)
raw_fh = open(text["raw"], "r")
line_count = 0
# read through the object file, pass the words to treetagger
for line in raw_fh:
type, word, id, attrib = line.split('\t')
id = id.split()
if type == "word":
word = word.decode('utf-8', 'ignore').lower().encode('utf-8')
# close and re-open the treetagger process to prevent garbage output.
if line_count > maxlines:
tt_worker.stdin.close()
tt_worker.wait()
new_ttout_fh = open(text["raw"] + ".ttout", "a")
tt_worker = Popen(tt_args, stdin=PIPE, stdout=new_ttout_fh)
line_count = 0
print >> tt_worker.stdin, word
line_count += 1
# finish tagging
tt_worker.stdin.close()
tt_worker.wait()
# go back through the object file, and add the treetagger results to each word
tmp_fh = open(text["raw"] + ".tmp", "w")
tag_fh = open(text["raw"] + ".ttout", "r")
for line in open(text["raw"], "r"):
type, word, id, attrib = line.split('\t')
id = id.split()
record = Record(type, word, id)
record.attrib = eval(attrib)
if type == "word":
tag_l = tag_fh.readline()
next_word, tag = tag_l.split("\t")[0:2]
pos, lem, prob = tag.split(" ")
if next_word != word.decode('utf-8',
'ignore').lower().encode('utf-8'):
print >> sys.stderr, "TREETAGGER ERROR:", next_word, " != ", word, pos, lem
return
else:
record.attrib["pos"] = pos
record.attrib["lemma"] = lem
print >> tmp_fh, record
else:
print >> tmp_fh, record
os.remove(text["raw"])
os.rename(text["raw"] + ".tmp", text["raw"])
os.remove(text["raw"] + ".ttout")
def fix_pages(loader_obj,text,depth=4):
"""Unfinished, do not use"""
object_types = ['doc', 'div1', 'div2', 'div3', 'para', 'sent', 'word'][:depth]
current_page = 0;
temp_file = open(text["sortedtoms"] + ".tmp","w")
for line in open(text["sortedtoms"]):
type, word, id, attrib = line.split('\t')
id = id.split()
record = Record(type, word, id)
record.attrib = eval(attrib)
def test_serialize(self):
tests = [{
'body': {},
'expected': '{}'
}, {
'body': {
'name': 'siri',
'age': 1
},
'expected': '{"name": "siri", "age": 1}'
}
# TODO: deal with unicode
]
for t in tests:
resp = mock.Mock()
resp.body = t['body']
serialize(dummy, resp, dummy)
self.assertDictEqual(eval(resp.body), eval(t['expected']))
def get(self):
ressource_options = default_ressource_options(request, current_app)
cursor = current_app.mongo.feeds
spec = {}
# TODO: check the query
if ressource_options['query']:
spec = eval(ressource_options['query'])
data = [feed for feed in cursor.find(spec=spec)]
return dict({'data': data}, **ressource_options)
def word_counts_table(loader_obj, obj_type='doc'):
object_types = ['doc', 'div1', 'div2', 'div3', 'para', 'sent', 'word']
depth = object_types.index(obj_type) + 1 ## this is for philo_id slices
## Retrieve column names from toms.db
original_fields = ['philo_type', 'philo_name', 'philo_id', 'philo_seq', '%s_token_count' % obj_type]
toms_conn = sqlite3.connect(loader_obj.destination + '/toms.db')
toms_conn.row_factory = sqlite3.Row
toms_c = toms_conn.cursor()
toms_c.execute('select * from toms')
extra_fields = [i[0] for i in toms_c.description if i[0] not in original_fields]
field_list = original_fields + extra_fields + ['bytes']
## Create table
conn = sqlite3.connect(loader_obj.destination + '/%s_word_counts.db' % obj_type)
conn.text_factory = str
c = conn.cursor()
columns = ','.join(field_list)
query = 'create table if not exists toms (%s)' % columns
c.execute(query)
c.execute('create index word_index on toms (philo_name)')
c.execute('create index philo_id_index on toms (philo_id)')
conn.commit()
file_in = '%s/%s_word_counts_sorted' % (loader_obj.workdir, obj_type)
sequence = 0
for line in open(file_in):
(philo_type,philo_name,id,attrib) = line.split("\t",3)
philo_id = ' '.join(id.split()[:depth])
philo_id = philo_id + ' ' + ' '.join('0' for i in range(7 - depth))
row = {}
row["philo_type"] = philo_type
row["philo_name"] = philo_name
row["philo_id"] = philo_id
row["philo_seq"] = sequence
attrib = eval(attrib)
## Fetch missing fields from toms.db
toms_query = 'select %s from toms where philo_id=?' % ','.join(extra_fields)
toms_c.execute(toms_query, (philo_id,))
results = [i for i in toms_c.fetchone()]
row.update(dict(zip(extra_fields, results)))
for k in attrib:
row[k] = attrib[k]
row_key = []
row_value = []
for k,v in row.items():
row_key.append(k)
row_value.append(v)
key_string = "(%s)" % ",".join(x for x in row_key)
insert = "INSERT INTO toms %s values (%s);" % (key_string,",".join("?" for i in row_value))
c.execute(insert,row_value)
sequence += 1
conn.commit()
def run(self):
"""
Update all tests found in the files given by the `test_suffixes` class attribute.
"""
# import all the test_suffix experiment files using SourceFileLoader
# adapted from: http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path
for test_suffix in self.test_suffixes:
test_module_path = join(
self.tests_directory,
'test_experiment_{}.py'.format(test_suffix))
test_module = SourceFileLoader('loaded_{}'.format(test_suffix),
test_module_path).load_module()
# skip the module if it tells us that it doesn't want the data for its tests updated
if hasattr(test_module, '_AUTO_UPDATE'):
if not test_module._AUTO_UPDATE:
continue
# iterate over all the members and focus on only the experiment functions
# but skip over the functions that are decorated with '@raises' since those
# functions do not need any test data to be updated. For the rest, try to get
# the source since that's what we need to update the test files
for member_name, member_object in getmembers(test_module):
if isfunction(member_object) and member_name.startswith(
'test_run_experiment'):
function = member_object
# get the qualified name of the member function
member_qualified_name = member_object.__qualname__
# check if it has 'raises' in the qualified name and skip it, if it does
if 'raises' in member_qualified_name:
continue
# otherwise first we check if it's the parameterized function and if so
# we can easily get the source from the parameter list
if member_name.endswith('parameterized'):
for param in function.parameterized_input:
source_name = param.args[0]
skll = param.kwargs.get('skll', False)
self.update_source(source_name, skll=skll)
# if it's another function, then we actually inspect the code
# to get the source. Note that this should never be a SKLL experiment
# since those should always be run parameterized
else:
function_code_lines = getsourcelines(function)
source_line = [
line for line in function_code_lines[0]
if re.search(r'source = ', line)
]
source_name = eval(
source_line[0].strip().split(' = ')[1])
self.update_source(source_name)
def loadmongoconfig():
mdb_transforms = mdb.collection('transforms')
mdb_filters = mdb.collection('filters')
mdb_selectors = mdb.collection('selectors')
mdb_publishers = mdb.collection('publishers')
rules = []
for tr in mdb_transforms.find():
if DBG >= 1: logger.debug("transform: %s", tr)
if tr['active'] != 'on':
logger.info("transform '%s' not active" % tr['transform_name'])
continue
te = {'name': tr['transform_name']}
fil = mdb_filters.find_one({'_id': tr['filter']})
if fil == None:
raise SLException("no filter obj %s for transform %s" %
(tr['filter'], tr['transform_name']))
te[fil['filter_name']] = eval(fil['filter_string'])
sel = mdb_selectors.find_one({'_id': tr['selector']})
if sel == None:
raise SLException("no selector obj %s for transform %s" %
(tr['selector'], tr['transform_name']))
nsel = eval(sel['selector_string'])
if len(nsel) != 0:
te[sel['selector_name']] = eval(sel['selector_string'])
msel = sel['selector_name']
else:
msel = ''
pub = mdb_publishers.find_one({'_id': tr['publisher']})
if pub == None:
raise SLException("no publisher obj %s for transform %s" %
(tr['publisher'], tr['transform_name']))
te[pub['publisher_name']] = eval(pub['publisher_string'])
te['rules'] = [[fil['filter_name'], msel, pub['publisher_name']]]
rules.append(te)
# mdb_nodes = mdb.collection(':
return rules
def get_frequency(environ, start_response):
status = '200 OK'
headers = [('Content-type', 'application/json; charset=UTF-8'),
("Access-Control-Allow-Origin", "*")]
start_response(status, headers)
config = WebConfig(os.path.abspath(os.path.dirname(__file__)).replace('scripts', ''))
db = DB(config.db_path + '/data/')
request = WSGIHandler(environ, config)
setattr(request, 'frequency_field', simplejson.dumps(
eval('"%s"' % request.frequency_field)))
results = frequency_results(request, config, sorted=True)
yield simplejson.dumps(results)
def normalize_unicode_raw_words(loader_obj, text):
tmp_file = open(text["raw"] + ".tmp", "w")
for line in open(text["raw"]):
rec_type, word, id, attrib = line.split('\t')
id = id.split()
if rec_type == "word":
word = word.decode("utf-8").lower().encode("utf-8")
record = Record(rec_type, word, id)
record.attrib = eval(attrib)
print >> tmp_file, record
os.remove(text["raw"])
os.rename(text["raw"] + ".tmp", text["raw"])
def return_point_by_id(self, point_id, retpoint):
##do the select
##consider indexing the table by point id.
self.cur_prev.execute("select configpoint, calcsteps, lambda from points where myid = ?", [str(point_id)])
##save the info
r = self.cur_prev.fetchone()
# retpoint.append(ast.literal_eval(str(r[0])))
retpoint.append(ast.eval(str(r[0])))
return r[1], r[2]
def normalize_unicode_raw_words(loader_obj, text):
tmp_file = open(text["raw"] + ".tmp","w")
for line in open(text["raw"]):
rec_type, word, id, attrib = line.split('\t')
id = id.split()
if rec_type == "word":
word = word.decode("utf-8").lower().encode("utf-8")
record = Record(rec_type, word, id)
record.attrib = eval(attrib)
print >> tmp_file, record
os.remove(text["raw"])
os.rename(text["raw"] + ".tmp",text["raw"])
def latest_version():
'''Returns the latest version number or None if a problem occurs.'''
try:
# Try reading the repository from GitHub.
# Request module is required.
from base64 import b64decode
from requests import get, codes
repo_name = None
request = None
token = None
user = None
# Read local information about the repository.
try:
with data('repo.pydef') as file:
repo: dict = eval(file.read())
user = repo['user']
repo_name = repo['name']
except OSError:
return print('File repo.pydef not found.')
except:
return print('Trouble parsing repo dictionary.')
# Try to read local token.
# Token is necessarry if the repository is private.
try:
with data('.token') as file:
token = file.read().rstrip('\n')
except OSError:
print('Token not present. Assuming that the repository is public.')
# Build url and send request through GitHub API.
url = ('https://api.github.com/repos/'
f'{user}/{repo_name}/'
'contents/data/version.pydef')
if token is None:
# Assume that repository is public. No header needed.
request = get(url)
else:
# Token is available. Send it inside the Authorization header.
request = get(url, headers={'Authorization': f'token {token}'})
if request.status_code == codes.ok: #pylint: disable=no-member
print('Reading latest version from github.')
# Before we can read the content, we have to decode it.
return int(b64decode(request.json()['content']))
else:
return print('Repository or file not found.')
except ImportError:
return print("Module 'requests' not found.")
def parse_values(input, size):
"""Parse fuzzy values for the aggregation axis.
The input comes from ConfigObj as a list of strings::
>>> print parse_values(["10", "20", "..."], 5)
[ 10. 20. 30. 40. 50.]
>>> print parse_values(["1", "...", "10"], 5)
[ 1. 3.25 5.5 7.75 10. ]
>>> print parse_values(["1", "1", "2", "3", "5"], 5)
[1 1 2 3 5]
Returns a list with the values.
"""
if len(input) == size and "..." not in input:
return np.asarray(map(eval, input))
start, next, stop = input[:]
if next == '...':
return np.linspace(eval(start), eval(stop), size)
elif stop == '...':
dx = eval(next)-eval(start)
return np.arange(eval(start), eval(start)+size*dx, dx)
else:
raise Exception('Unable to parse: %s' % input)
def tag_words(loader_obj,text):
# Set up the treetagger process
tt_args = [tt_path,"-token","-lemma","-prob",'-no-unknown', "-threshold",".01",param_file]
ttout_fh = open(text["raw"]+".ttout","w")
tt_worker = Popen(tt_args,stdin=PIPE,stdout=ttout_fh)
raw_fh = open(text["raw"],"r")
line_count = 0
# read through the object file, pass the words to treetagger
for line in raw_fh:
type, word, id, attrib = line.split('\t')
id = id.split()
if type == "word":
word = word.decode('utf-8', 'ignore').lower().encode('utf-8')
# close and re-open the treetagger process to prevent garbage output.
if line_count > maxlines:
tt_worker.stdin.close()
tt_worker.wait()
new_ttout_fh = open(text["raw"]+".ttout","a")
tt_worker = Popen(tt_args, stdin=PIPE,stdout=new_ttout_fh)
line_count = 0
print >> tt_worker.stdin, word
line_count += 1
# finish tagging
tt_worker.stdin.close()
tt_worker.wait()
# go back through the object file, and add the treetagger results to each word
tmp_fh = open(text["raw"]+".tmp","w")
tag_fh = open(text["raw"] + ".ttout","r")
for line in open(text["raw"],"r"):
type, word, id, attrib = line.split('\t')
id = id.split()
record = Record(type,word,id)
record.attrib = eval(attrib)
if type == "word":
tag_l = tag_fh.readline()
next_word,tag = tag_l.split("\t")[0:2]
pos,lem,prob = tag.split(" ")
if next_word != word.decode('utf-8', 'ignore').lower().encode('utf-8'):
print >> sys.stderr, "TREETAGGER ERROR:",next_word," != ",word,pos,lem
return
else:
record.attrib["pos"] = pos
record.attrib["lemma"] = lem
print >> tmp_fh, record
else:
print >> tmp_fh, record
os.remove(text["raw"])
os.rename(text["raw"] + ".tmp",text["raw"])
os.remove(text["raw"] + ".ttout")
def return_point_by_id(self, point_id, retpoint):
##do the select
##consider indexing the table by point id.
self.cur_prev.execute(\
'select configpoint, calcsteps, lambda from points where myid = ?', [str(point_id)])
##save the info
r = self.cur_prev.fetchone()
#retpoint.append(ast.literal_eval(str(r[0])))
retpoint.append(ast.eval(str(r[0])))
return r[1], r[2]
def loadrules(self):
def calldb(collection, func, what=None):
q = {'func': func, 'collection': collection}
q['what'] = what
return self.engine.ask_question("SteamDB", None, q)
rules = []
for tr in calldb('transforms', 'find'):
if tr['active'] != 'on':
self.logger.warn("transform '%s' not active",
tr['transform_name'])
continue
te = {'name': tr['transform_name']}
fil = calldb('filters', 'find_one', {'_id': tr['filter']})
if fil == None:
raise SLException("no filter obj %s for transform %s" %
(tr['filter'], tr['transform_name']))
te[fil['filter_name']] = eval(fil['filter_string'])
sel = calldb('selectors', 'find_one', {'_id': tr['selector']})
if sel == None:
raise SLException("no selector obj %s for transform %s" %
(tr['selector'], tr['transform_name']))
nsel = eval(sel['selector_string'])
if len(nsel) != 0:
te[sel['selector_name']] = eval(sel['selector_string'])
msel = sel['selector_name']
else:
msel = ''
pub = calldb('publishers', 'find_one', {'_id': tr['publisher']})
if pub == None:
raise SLException("no publisher obj %s for transform %s" %
(tr['publisher'], tr['transform_name']))
te[pub['publisher_name']] = eval(pub['publisher_string'])
te['rules'] = [[fil['filter_name'], msel, pub['publisher_name']]]
rules.append(te)
return rules
def read_info():
if REPO._INFO_STATUS is None:
path, file = fdata_('info.pydef')
try:
with file:
info = eval(file.read())
REPO.USER = info['user']
REPO.NAME = info['repo_name']
REPO.PRIVATE = info['private']
REPO._INFO_STATUS = True
except OSError:
print(f'File {path} not found.')
REPO._INFO_STATUS = False
return REPO._INFO_STATUS
def rgb_to_hex(color):
"""
Converts from rgb to hex
Parameters:
-----------
color : string
Color representation on hex or rgb
Example:
rgb_to_hex('rgb(23,25,24)')
"""
rgb=eval(color.replace('rgb',''))
return '#' + ''.join(['{0:02x}'.format(x) for x in rgb])
def checkForCoreFiles(self):
try:
self.coreSearch = self.run(
'%s ls -lrt %s' %
(self.setsudo, self.coresearch), sudo=self.sudo)
if len(self.coreSearch) == 0:
self.coreSearch = False
elif self.coreSearch.split()[1] == "cannot" and self.coreSearch.split()[2] == "access":
self.coreSearch = False
else:
self.coreSearch = eval(self.coreSearch)
except BaseException:
#print "problem in tracing the core files.Please check manually for the existence of core file."
self.coreSearch = False
def process_corpus(self):
lines = [eval(line.strip()) for line in open("tweetlabels.txt", "r")]
stop = stopwords.words("english")
stop.remove("no")
corpus = [x[0] for x in lines]
labels = [x[1] for x in lines]
pr = Processor.Processor(stop)
normalized_corpus, lines = pr.process(corpus)
del lines
tfidf = Tfidf(ngram_range=(1,2), binary=True,
tokenizer=tweet_tokenizer)
X = tfidf.fit_transform(normalized_corpus)
self.X = X
self.labels = labels
def collocation_fetcher(environ,start_response):
status = '200 OK'
headers = [('Content-type', 'application/json; charset=UTF-8'),("Access-Control-Allow-Origin","*")]
start_response(status,headers)
environ["SCRIPT_FILENAME"] = environ["SCRIPT_FILENAME"].replace('scripts/collocation_fetcher.py', '')
cgi = urlparse.parse_qs(environ["QUERY_STRING"],keep_blank_values=True)
full_report = eval(cgi.get('full_report',['True'])[0])
db, path_components, q = parse_cgi(environ)
hits = db.query(q["q"],q["method"],q["arg"],**q["metadata"])
if full_report:
all_colloc, left_colloc, right_colloc = r.fetch_collocation(hits, environ["SCRIPT_FILENAME"], q, db)
yield dumps([all_colloc, left_colloc, right_colloc])
else:
results = r.fetch_collocation(hits, environ["SCRIPT_FILENAME"], q, db, full_report=False)
yield dumps(results)
def init_from_str(self):
r = r"([a-zA-Z]*)=([^,)]*)"
opt_str = self.param.lower()
kwargs = dict([(i, eval(j)) for (i, j) in re.findall(r, opt_str)])
if "sgd" in opt_str:
optimizer = SGD(**kwargs)
elif "adagrad" in opt_str:
optimizer = AdaGrad(**kwargs)
elif "rmsprop" in opt_str:
optimizer = RMSProp(**kwargs)
elif "adam" in opt_str:
optimizer = Adam(**kwargs)
else:
raise NotImplementedError("{}".format(opt_str))
return optimizer
def main():
"""
Indexes MongoDB
"""
parser = init_parser(description)
args = parser.parse_args()
# 把读进来的配置文件转成字典
config = eval(args.config.read())
log.debug("\n")
the_mongo = MONGODB
if isinstance(the_mongo, dict):
process_index('mongodb', the_mongo, config)
process_index('log_mongodb', the_mongo, config)
def main():
"""
Indexes MongoDB
"""
parser = init_parser(description)
args = parser.parse_args()
# 把读进来的配置文件转成字典
config = eval(args.config.read())
log.debug("\n")
the_mongo = MONGODB
if isinstance(the_mongo, dict):
process_index('mongodb', the_mongo, config)
process_index('log_mongodb', the_mongo, config)
def run(self):
"""
Update all tests found in the files given by the `test_suffixes` class attribute.
"""
# import all the test_suffix experiment files using SourceFileLoader
# adapted from: http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path
for test_suffix in self.test_suffixes:
test_module_path = join(self.tests_directory, 'test_experiment_{}.py'.format(test_suffix))
test_module = SourceFileLoader('loaded_{}'.format(test_suffix), test_module_path).load_module()
# skip the module if it tells us that it doesn't want the data for its tests updated
if hasattr(test_module, '_AUTO_UPDATE'):
if not test_module._AUTO_UPDATE:
continue
# iterate over all the members and focus on only the experiment functions
# but skip over the functions that are decorated with '@raises' since those
# functions do not need any test data to be updated. For the rest, try to get
# the source since that's what we need to update the test files
for member_name, member_object in getmembers(test_module):
if isfunction(member_object) and member_name.startswith('test_run_experiment'):
function = member_object
# get the qualified name of the member function
member_qualified_name = member_object.__qualname__
# check if it has 'raises' in the qualified name and skip it, if it does
if 'raises' in member_qualified_name:
continue
# otherwise first we check if it's the parameterized function and if so
# we can easily get the source from the parameter list
if member_name.endswith('parameterized'):
for param in function.parameterized_input:
source_name = param.args[0]
skll = param.kwargs.get('skll', False)
self.update_source(source_name, skll=skll)
# if it's another function, then we actually inspect the code
# to get the source. Note that this should never be a SKLL experiment
# since those should always be run parameterized
else:
function_code_lines = getsourcelines(function)
source_line = [line for line in function_code_lines[0]
if re.search(r'source = ', line)]
source_name = eval(source_line[0].strip().split(' = ')[1])
self.update_source(source_name)
def inner_make_object_ancestors(loader_obj, text):
temp_file = text['words'] + '.tmp'
output_file = open(temp_file, 'w')
for line in open(text['words']):
type, word, id, attrib = line.split('\t')
id = id.split()
record = Record(type, word, id)
record.attrib = eval(attrib)
for type in types:
zeros_to_add = ['0' for i in range(7 - type_depth[type])]
philo_id = id[:type_depth[type]] + zeros_to_add
record.attrib[type + '_ancestor'] = ' '.join(philo_id)
print >> output_file, record
output_file.close()
os.remove(text['words'])
os.rename(temp_file, text['words'])
def init_from_str(self):
r = r"([a-zA-Z]*)=([^,)]*)"
sch_str = self.param.lower()
kwargs = dict([(i, eval(j)) for (i, j) in re.findall(r, sch_str)])
if "constant" in sch_str:
scheduler = ConstantScheduler(**kwargs)
elif "exponential" in sch_str:
scheduler = ExponentialScheduler(**kwargs)
elif "noam" in sch_str:
scheduler = NoamScheduler(**kwargs)
elif "king" in sch_str:
scheduler = KingScheduler(**kwargs)
else:
raise NotImplementedError("{}".format(sch_str))
return scheduler
def inner_make_object_ancestors(loader_obj, text):
temp_file = text['words'] + '.tmp'
output_file = open(temp_file, 'w')
for line in open(text['words']):
type, word, id, attrib = line.split('\t')
id = id.split()
record = Record(type, word, id)
record.attrib = eval(attrib)
for type in types:
zeros_to_add = ['0' for i in range(7 - type_depth[type])]
philo_id = id[:type_depth[type]] + zeros_to_add
record.attrib[type + '_ancestor'] = ' '.join(philo_id)
print >> output_file, record
output_file.close()
os.remove(text['words'])
os.rename(temp_file, text['words'])
def game_loop(board, piece):
print()
print_board(board)
while (True):
try:
move = eval(input('Place %s where? ' % piece))
move = tuple(reversed(move))
# x,y -> y,x (easier to use)
if is_valid_move(board, piece, move):
place_piece(board, piece, move)
return
else:
raise AssertionError
except (TypeError, ValueError, IndexError, SyntaxError,
AssertionError):
# ------------------bad input------------------ ---bad move---
print('Invalid move. Try again.')
def make_sql_table(self, table, file_in, indices=[], depth=7):
conn = self.dbh
c = conn.cursor()
columns = 'philo_type,philo_name,philo_id,philo_seq'
query = 'create table if not exists %s (%s)' % (table, columns)
c.execute(query)
print "%s table in toms.db database file..." % table,
sequence = 0
for line in open(file_in):
philo_type, philo_name, id, attrib = line.split("\t",3)
fields = id.split(" ",8)
if len(fields) == 9:
row = {}
philo_id = " ".join(fields[:depth])
row["philo_type"] = philo_type
row["philo_name"] = philo_name
row["philo_id"] = philo_id
row["philo_seq"] = sequence
row.update(eval(attrib))
columns = "(%s)" % ",".join([i for i in row])
insert = "INSERT INTO %s %s values (%s);" % (table, columns,",".join(["?" for i in row]))
values = [v for k,v in row.items()]
try:
c.execute(insert,values)
except sqlite3.OperationalError:
c.execute("PRAGMA table_info(%s)" % table)
column_list = [i[1] for i in c.fetchall()]
for column in row:
if column not in column_list:
c.execute("ALTER TABLE %s ADD COLUMN %s;" % (table,column))
c.execute(insert,values)
sequence += 1
conn.commit()
for index in indices:
try:
c.execute('create index if not exists %s_%s_index on %s (%s)' % (table,index,table,index))
except sqlite3.OperationalError:
pass
conn.commit()
print 'done.'
if self.clean:
os.system('rm %s' % file_in)
def restore(self, obj):
try:
restore_method = [
m for m in self.methods if isinstance(obj, m.type)
]
if not restore_method:
restore_method = [
m for m in self.methods if obj.startswith(m.name + '-->')
]
if not restore_method:
restore_method = [
t for t in ntypes if obj.startswith(str(t)[8:-2] + '-->')
]
if restore_method:
return eval(obj[len(str(restore_method[0])) - 7:])
return restore_method[0].defreezer(obj, self)
except:
return obj
def main():
"""
Indexes MongoDB
"""
parser = init_parser(description)
args = parser.parse_args()
# 把读进来的配置文件转成字典
config = eval(args.config.read())
if isinstance(args.target, list):
for target in args.target:
log.debug("切换环境： %s" % target)
log.debug("\n")
the_mongo = MONGODB[target]
if isinstance(the_mongo, dict):
process_index('mongodb', the_mongo, config)
process_index('log_mongodb', the_mongo, config)
def smash_these_unicode_columns(loader_obj, text):
tmp_file = open(text["sortedtoms"] + ".tmp","w")
for line in open(text["sortedtoms"]):
type, word, id, attrib = line.split('\t')
id = id.split()
record = Record(type, word, id)
record.attrib = eval(attrib)
for column in columns:
if column in record.attrib:
# print >> sys.stderr, repr(record.attrib)
col = record.attrib[column].decode("utf-8")
col = col.lower()
smashed_col = [c for c in unicodedata.normalize("NFKD",col) if not unicodedata.combining(c)]
record.attrib[column + "_norm"] = ''.join(smashed_col).encode("utf-8")
#record.attrib[column + "_norm"] = ''.join([c.encode("utf-8") for c in unicodedata.normalize('NFKD',record.attrib[column].decode("utf-8").lower()) if not unicodedata.combining(c)])
print >> tmp_file, record
tmp_file.close()
os.remove(text["sortedtoms"])
os.rename(text["sortedtoms"] + ".tmp",text["sortedtoms"])
def make_word_counts(loader_obj, text, depth=4):
object_types = ['doc', 'div1', 'div2', 'div3', 'para', 'sent', 'word']
counts = [0 for i in range(depth)]
temp_file = text['raw'] + '.tmp'
output_file = open(temp_file, 'w')
for line in open(text['raw']):
type, word, id, attrib = line.split('\t')
id = id.split()
record = Record(type, word, id)
record.attrib = eval(attrib)
for d, count in enumerate(counts):
if type == 'word':
counts[d] += 1
elif type == object_types[d]:
record.attrib['word_count'] = counts[d]
counts[d] = 0
print >> output_file, record
output_file.close()
os.remove(text['raw'])
os.rename(temp_file, text['raw'])
def normalize(self, dataset, datatype="normal"):
print("Normalizing..")
if datatype == "normal":
if len(
dataset[0]
) > self.trainingsize: # check if new data is larger than training sample(it will be just one set, as a list)
self.addtotraining(len(dataset[0]))
self.train("retrain")
newdataset = []
for data in dataset:
#if type(data) == str:
data = eval(str(data))
newdataset.append(data)
if len(data) > self.datasize:
self.datasize = len(data)
dataset = newdataset
trainingdata = Settings().getdata("MACHINELEARNING",
"trainingfeatures")
for data in dataset:
print("comparing datalength!")
while len(data) != self.datasize:
if len(data) < self.datasize:
print("new data is smaller!")
data.append(9999)
else:
print("new data is larger!")
for tdata in trainingdata:
while len(data) != len(tdata):
tdata.append(9999)
if datatype == "training":
self.trainingsize = self.datasize # set training sample size
Settings().setdata(
{"MACHINELEARNING": {
"trainingsize": self.trainingsize
}})
self.trainingfeaturesaslist = dataset
Settings().setdata(
{"MACHINELEARNING": {
"normalizeddataset": dataset
}})
return dataset
def read_footer(ifolder=None,tfile=None,ext='all'):
from sq_tools import read_footer
from ast import literal_eval as eval
if ext=='all':
ext=['nsp','nsx','nsz','xci','xcz']
else:
if isinstance(ext, list):
ext=ext
else:
try:
ext=eval(ext)
except:pass
ext=ext.split(',')
if ifolder!=None:
files=listmanager.folder_to_list(ifolder,ext)
elif tfile!=None:
files=read_lines_to_list(tfile,all=True)
else:
return False
for filepath in files:
read_footer(filepath)
def word_frequencies_per_obj(loader_obj, text, depth=1):
object_types = ['doc', 'div1', 'div2', 'div3', 'para', 'sent',
'word'][:depth]
files_path = loader_obj.destination + '/WORK/'
try:
os.mkdir(files_path)
except OSError:
## Path was already created
pass
for d, obj in enumerate(object_types):
file = text['name'] + '.%s.sorted' % obj
output = open(files_path + file, 'w')
d = d + 1
old_philo_id = []
records = {}
for line in open(text['words']):
type, word, id, attrib = line.split('\t')
attrib = eval(attrib)
philo_id = id.split()
record = Record(type, word, philo_id)
count_key = obj + '_token_count'
byte = attrib['byte_start']
del attrib['byte_start']
record.attrib = {count_key: attrib[count_key]}
if philo_id[:d] != old_philo_id[:d]:
if records:
for w in records:
print >> output, records[w]
records = {}
if word not in records:
record.attrib['bytes'] = []
record.attrib['bytes'] = str(byte)
records[word] = record
else:
records[word].attrib['bytes'] += ' ' + str(byte)
old_philo_id = philo_id
for w in records:
print >> output, records[w]
output.close()
