def getDailyWeatherDataForPrevision(tempMean, t):
db = sqlite3.connect(db_path_string)
cursor = db.cursor()
t_day = int(t - (t % (24*3600))) - 3600 # weather uses UTC + 2 time
t_next_day = t_day + 3600*24
#print 'timestamp for daily weather', t_day
data = cursor.execute('SELECT avg(temperatur), sum(precipitation) FROM weatherForecast WHERE timestamp>=:t_min and timestamp<=:t_max',\
{"t_min":t_day, "t_max":t_next_day}).fetchone()
if not util.is_number(data[0]):
#print 'no temperature data for this day, assuming average temperature'
dayTemperatureAvg = tempMean
else:
dayTemperatureAvg = float(data[0])
if not util.is_number(data[1]):
#print 'no precipitation data for this day, assuming no precipitations'
dayPrecipitationTotal = 0.0
else:
dayPrecipitationTotal = float(data[1])
db.close()
return [dayTemperatureAvg, dayPrecipitationTotal]
def eq(self, a, b):
''' operator "="
If "=" is applied to two equal blocks or two equal integers,
then the result equals 1, otherwise 0'''
if not (util.is_number(a) and util.is_number(b)) and not(self.is_block(a) and self.is_block(b)):
raise ValueError("Operator '=' can only be applied to two blocks or two integers!")
return a == b
def write_table(self,path):
if not self.page_tables: return
# page sort
sort_page = [p for p in self.page_tables.keys()]
sort_page.sort()
# set cell covers
i = 1
for page in sort_page:
tables = self.page_tables[page]
self.page_tables[page] = []
for t in tables:
info = self.get_table_info(t,page)
tt = table_structure.Table(t)
tt.info = info
tt.set_table_cells(i)
# pdb.set_trace()
i += tt.size[0]+2
self.page_tables[page].append(tt)
# write tables in excel
wb = xlwt.Workbook()
sheet = wb.add_sheet('tables',cell_overwrite_ok=True)
j = 0
for page in sort_page:
tables = self.page_tables[page]
for table in tables:
sheet.write(j,0,"page "+str(page+1))
sheet.write(j,1,table.info)
j += table.size[0]+2
for cell in table.cells:
if cell.covers:
if len(cell.covers)>1:
top_row = cell.covers[0][0]
bottom_row = cell.covers[-1][0]
left_column = cell.covers[0][1]
right_column = cell.covers[-1][1]
if util.is_number(cell.text): # if cell.text is number, save in number style
content = cell.text.replace(',','')
if content.isdigit():
sheet.write_merge(top_row, bottom_row, left_column, right_column, float(content))
else:
sheet.write_merge(top_row, bottom_row, left_column, right_column, cell.text)
else:
sheet.write_merge(top_row, bottom_row, left_column, right_column, cell.text)
else:
if util.is_number(cell.text):
content = cell.text.replace(',','')
if content.isdigit():
sheet.write(cell.covers[0][0],cell.covers[0][1],float(content))
else:
sheet.write(cell.covers[0][0],cell.covers[0][1],cell.text)
else:
sheet.write(cell.covers[0][0],cell.covers[0][1],cell.text)
wb.save(path)
def group(self):
''' operation "g" '''
arg1 = self.data.pop() # the argument on top of the stack will be at the end of the block
arg2 = self.data.pop()
if self.is_block(arg1) and self.is_block(arg2):
# two blocks
block = arg2[0:len(arg2) - 1]
arg1 = arg1[1:len(arg1)]
if util.is_number(arg2[-2]) and util.is_number(arg1[0]):
block += " "
block += arg1
elif util.is_number(arg1) and util.is_number(arg2):
# two integers
block = "[" + str(arg2) + " " + str(arg1) + "]"
elif util.is_number(arg2) and self.is_block(arg1):
# interger (arg2) and block (arg1)
block = "[" + str(arg2)
arg1 = arg1[1:len(arg1)]
if util.is_number(arg1[0]):
block += " "
block += arg1
elif self.is_block(arg2) and util.is_number(arg1):
# block (arg2) and integer (arg1)
block = arg2[0:len(arg2) - 1]
if util.is_number(arg2[-2]):
block += " "
block += str(arg1) + "]"
else:
raise ValueError("todo")
#TODO: remove debug
# print "group block: '{0}'".format(block)
self.data.push(block)
def getVacationData():
#period of the holiday data :2014-2015
start = int(util.datetimeToTimestamp(datetime.datetime(2014,01,01)))
end = int(util.datetimeToTimestamp(datetime.datetime(2015,12,31)))
step = 3600 * 24
vacation = dict()
with vacation_data_path.open('r') as f:
read_data = f.read()
vacationDayIds = read_data.split(';')
# add all vacation days
for dayId in vacationDayIds:
if util.is_number(dayId):
dayId = timestampToDayId(float(dayId))
vacation[dayId] = 1
# add remaining days
for dayId2 in range(start , end, step):
dayId2 = timestampToDayId(float(dayId2))
if not dayId2 in vacation:
vacation[dayId2] = 0
return vacation
def apply_crypt(string_const):
"""Apply the crypto routine to the string costant"""
if string_const is None:
return ''
if not u.is_number(string_const):
return 'a7ign' + u.crypt_string(string_const)
else:
return string_const
def getDailyWeatherData():
db = sqlite3.connect(db_path_string)
cursor = db.cursor()
weather_data = cursor.execute('SELECT * FROM weather_day').fetchall()
temperatures, precipitations = list(), list()
#temperatureTotal, temperatureCount, precipitationTotal, precipitationCount = 0 , 0 , 0 , 0
for data in weather_data:
if util.is_number(data[1]) :
#temperatureTotal = temperatureTotal + float(data[1])
#temperatureCount = temperatureCount + 1
temperatures.append(float(data[1]))
if util.is_number(data[2]) :
#precipitationTotal = precipitationTotal + float(data[2])
#precipitationCount = precipitationCount + 1
precipitations.append(float(data[2]))
tempMean = numpy.mean(temperatures)
tempStdDev = numpy.std(temperatures)
precStdDev = numpy.std(precipitations)
#print tempMean, tempStdDev, precStdDev
normalizedTemperatures, normalizedPrecipitations = dict(), dict()
for data2 in weather_data:
if util.is_number(data2[1]) :
#temperatureTotal = temperatureTotal + float(data2[1])
#temperatureCount = temperatureCount + 1
dayId = timestampToDayId(data2[0])
normalizedTemperatures[dayId] = (float(data2[1]) - tempMean ) / tempStdDev
if util.is_number(data2[2]) :
#precipitationTotal = precipitationTotal + float(data2[2])
#precipitationCount = precipitationCount + 1
dayId = timestampToDayId(data2[0])
normalizedPrecipitations[dayId] = float(data2[2]) / precStdDev
db.close()
return [tempMean, tempStdDev, precStdDev, normalizedTemperatures, normalizedPrecipitations]
def read(self):
# read from input stream
# item = chr(self.iS.read())
item = self.iS.read()
# only digits, commands and operators may be read
if util.is_number(item):
# self.data.push(int(item))
self.data.push(item)
elif item in self.operations:
raise ValueError("Input {0} is not a valid char code")
def write(self):
'''operation "w"'''
# make sure that the top element of the data stack is an integer or a command
val = self.data.pop()
# print ">> {0}".format(val)
if not util.is_number(val) and val not in self.operations:
raise ValueError("Cannot write to output stream" + val)
# for byte in to_ascii(val):
# self.oS.write(byte)
self.oS.write(val)
def get_tracform_meta(self, src, db=None):
"""
Returns the meta information about a form based on a form id (int or
long) or context (parent realm, parent id, TracForms subcontext).
"""
db = self._get_db(db)
cursor = db.cursor()
sql = """
SELECT id,
realm,
resource_id,
subcontext,
author,
time,
keep_history,
track_fields
FROM forms
"""
if not is_number(src):
sql += """
WHERE realm=%s
AND resource_id=%s
AND subcontext=%s
"""
else:
sql += """
WHERE id=%s
"""
form_id = None
realm = None
resource_id = None
subcontext = None
if not is_number(src):
realm, resource_id, subcontext = src
else:
form_id = src
src = tuple([src],)
cursor.execute(sql, src)
return cursor.fetchone() or \
(form_id, realm, resource_id, subcontext,
None, None, None, None)
def generate_features(name):
data = pandas.read_csv("homesite/{}.csv".format(name))
processed_data = pandas.DataFrame()
onehot_encode_fields = []
dropped_fields = ["PersonalField16", "PersonalField17", "PersonalField18", "PersonalField19"]
for k in data.keys():
if is_number(data[k][1]):
processed_data[k] = data[k]
elif k == "Original_Quote_Date":
months = []
years = []
for d in data[k]:
dt = datetime.datetime.strptime(d, "%Y-%m-%d")
months.append(dt.month)
years.append(dt.year)
processed_data["Month"] = pandas.Series(months)
processed_data["Year"] = pandas.Series(years)
else:
if k in dropped_fields:
continue
onehot_encode_fields.append(k)
df = encode_onehot(data, cols=onehot_encode_fields)
for k in df.keys():
if is_number(df[k][1]):
print k
processed_data[k] = df[k].apply(lambda x: locale.atof(x) if isinstance(x, basestring) else x)
elif k == "Original_Quote_Date":
months = []
years = []
for d in df[k]:
dt = datetime.datetime.strptime(d, "%Y-%m-%d")
months.append(dt.month)
years.append(dt.year)
processed_data["Month"] = pandas.Series(months)
processed_data["Year"] = pandas.Series(years)
processed_data.to_csv("homesite/processed_features{}.csv".format(name), quotechar='"')
def get_tracform_fields(self, src, db=None):
db = self._get_db(db)
cursor = db.cursor()
if is_number(src):
form_id = src
else:
form_id = self.get_tracform_meta(src, db=db)[0]
cursor.execute("""
SELECT field, author, time
FROM forms_fields
WHERE id=%s
""", (form_id,))
return cursor.fetchall()
def get_tracform_fields(self, src, db=None):
db = self._get_db(db)
cursor = db.cursor()
if is_number(src):
form_id = src
else:
form_id = self.get_tracform_meta(src, db=db)[0]
cursor.execute(
"""
SELECT field, author, time
FROM forms_fields
WHERE id=%s
""", (form_id, ))
return cursor.fetchall()
def getNearestWeatherPastForRegression(F, hourLimit, db_path):
db = sqlite3.connect(db_path)
cursor = db.cursor()
R, toRemove = dict(), list()
for time in F:
min_time = time-3600*hourLimit
data = cursor.execute('''SELECT timestamp, precipitation FROM Weather WHERE timestamp>:lowLimit AND timestamp<=:highLimit ORDER BY timestamp DESC''',{"lowLimit":min_time, "highLimit":time}).fetchone()
if data is None:
toRemove.append(int(time))
elif not util.is_number(data[1]):
toRemove.append(int(time))
else:
R[int(time)] = float(data[1])
return [R, toRemove]
def get_tracform_history(self, src, db=None):
db = self._get_db(db)
cursor = db.cursor()
if is_number(src):
form_id = src
else:
form_id = self.get_tracform_meta(src, db=db)[0]
cursor.execute("""
SELECT author, time, old_state
FROM forms_history
WHERE id=%s
ORDER BY time DESC
""", (form_id,))
return cursor.fetchall()
def create_simplified_projections():
for s in ['fanduel', 'draftkings']:
conf.site = s
conf.rotogrinder_hitter_path =\
'http://rotogrinders.com/projected-stats/mlb-hitter.csv?site=%s' % conf.site
conf.rotogrinder_pitcher_path =\
'http://rotogrinders.com/projected-stats/mlb-pitcher.csv?site=%s' % conf.site
sals_and_pos = retrieve_mlb_player_salaries_and_positions()
if conf.batting_orders is None:
conf.batting_orders, conf.pitchers = retrieve_mlb_batting_order()
df = pd.read_csv('./projections/%s_%s_%s.csv' %
(conf.site, conf.proj_date, conf.proj_iteration),
index_col=0)
idx = df.index[:]
means = []
for p in idx:
means.append([np.mean(df.loc[p][0:conf.simulated_game_count])])
mean_df = pd.DataFrame(means, index=idx, columns=['mean'])
df = df.join(mean_df)
# join our two data frames
projections_overall = df.join(sals_and_pos, how='left')
# drop players that weren't included
to_drop = []
for name in projections_overall.index:
if type(projections_overall.at[name, 'cost']) == np.ndarray:
print name
print projections_overall.loc[name]
pdb.set_trace()
if not is_number(projections_overall.at[name, 'cost']):
to_drop.append(name)
elif name in conf.excluded_batters or name in\
conf.excluded_pitchers:
to_drop.append(name)
elif conf.use_inclusion is False and projections_overall.at[
name, 'team'] in conf.excluded_teams:
to_drop.append(name)
elif conf.use_inclusion is True and projections_overall.at[
name, 'team'] not in conf.included_teams:
to_drop.append(name)
projections_overall = projections_overall.drop(to_drop)
projections_overall['exclude'] = 0
projections_overall[['pos', 'mean', 'cost', 'team', 'exclude']].to_csv(
'./simplified_projections/%s_%s_%s.csv' %
(conf.site, conf.proj_date, conf.proj_iteration))
def get_tracform_history(self, src, db=None):
db = self._get_db(db)
cursor = db.cursor()
if is_number(src):
form_id = src
else:
form_id = self.get_tracform_meta(src, db=db)[0]
cursor.execute(
"""
SELECT author, time, old_state
FROM forms_history
WHERE id=%s
ORDER BY time DESC
""", (form_id, ))
return cursor.fetchall()
def push_code(self, s):
'''push code onto the calculator's code stack.
Left-most command will be at the top of the code stacks'''
i = 0
l = []
while i < len(s):
if util.is_number(s[i]):
n = int(s[i])
while i + 1 < len(s) and util.is_number(s[i + 1]):
i = i + 1
n = n * 10 + int(s[i])
l.append(n)
elif s[i] == "[":
end = i
cnt = 1
while cnt != 0 or s[end] != "]":
end += 1
if s[end] == "[":
cnt += 1
elif s[end] == "]":
cnt -= 1
block = s[i:end + 1]
i = end
l.append(block)
elif s[i] in self.operations:
l.append(s[i])
elif s[i].isspace():
pass
else:
msg = "Invalid code : " + s[i:]
raise ValueError(msg)
i = i + 1
for i in reversed(l):
self.code.push(i)
def get_tracform_fieldinfo(self, src, field, db=None):
"""Retrieve author and time of last change per field."""
db = self._get_db(db)
cursor = db.cursor()
if is_number(src):
form_id = src
else:
form_id = self.get_tracform_meta(src, db=db)[0]
cursor.execute("""
SELECT author, time
FROM forms_fields
WHERE id=%s
AND field=%s
""", (form_id, field))
return cursor.fetchone() or (None, None)
def get_tracform_fieldinfo(self, src, field, db=None):
"""Retrieve author and time of last change per field."""
db = self._get_db(db)
cursor = db.cursor()
if is_number(src):
form_id = src
else:
form_id = self.get_tracform_meta(src, db=db)[0]
cursor.execute(
"""
SELECT author, time
FROM forms_fields
WHERE id=%s
AND field=%s
""", (form_id, field))
return cursor.fetchone() or (None, None)
def voting_matrix():
voting_matrix = []
for row in rollcall:
# just to stay consistent, we format the party as a list
mps.append((row[:36], [row[20:23]]))
votes = row[36:]
votes = [convert_sen(thing) for thing in votes if is_number(thing)]
voting_matrix.append(votes)
open(sen_filename_voting_mat, "w").close()
mat_f = open(sen_filename_voting_mat, "wb")
pickle.dump(voting_matrix, mat_f)
mat_f.close()
open(sen_filename_mps, "w").close()
mps_f = open(sen_filename_mps, "wb")
pickle.dump(mps, mps_f)
mps_f.close()
def process_line(self, l_info, unnamed=True):
if isinstance(l_info, str) and not is_number(l_info):
assert (Line(l_info) in self.lines)
return Line(l_info)
if not isinstance(l_info, tuple):
raise NotImplementedError(
f"[process_line] l_info must be tuple or string")
l_pred = l_info[0].lower()
l_args = l_info[1:]
l_val = None
if l_pred == "line":
assert (len(l_args) == 2)
ps = [self.process_point(p) for p in l_args]
l_val = FuncInfo("connecting", ps)
elif l_pred in ["perp-at", "para-at"]:
assert (len(l_args) == 2)
p = self.process_point(l_args[0])
l = self.process_line(l_args[1])
l_val = FuncInfo(l_pred, [p, l])
elif l_pred == "perp-bis":
assert (len(l_args) == 2)
ps = [self.process_point(p) for p in l_args]
l_val = FuncInfo(l_pred, ps)
elif l_pred in ["isogonal", "isotomic"]:
assert (len(l_args) == 4)
ps = [self.process_point(p) for p in l_args]
l_val = FuncInfo(l_pred, ps)
elif l_pred in ["i-bisector", "e-bisector"]:
assert (len(l_args) == 3)
ps = [self.process_point(p) for p in l_args]
l_val = FuncInfo(l_pred, ps)
elif l_pred == "reflect-ll":
assert (len(l_args) == 2)
ls = [self.process_line(l) for l in l_args]
l_val = FuncInfo(l_pred, ls)
if l_val is not None:
L = Line(l_val)
if unnamed:
self.unnamed_lines.append(L)
return L
else:
raise NotImplementedError(
f"[process_line] Unsupported line pred: {l_pred}")
def _get_mdf_filename(self):
if self.mosmask is not None:
mdffile = self.mosmask
# Expand the MOS mask number
if is_number(self.mosmask):
observatory, semester, prgtype, queuenum = self.get_program_id_parts()
mdffile = "%s%s%s%03i-%02i" % (observatory, semester, prgtype, queuenum, int(self.mosmask))
debug("...mosmask =", mdffile)
else:
mdffile = self.focal_plane_mask()
mdffile = fits_filename(mdffile)
#-----------------------------------------------------------------------
# Start searching around willy nilly for the MDF file
if os.path.exists(mdffile):
return mdffile
# note, the order in which directories are added to this list gives priority
dirs = []
if self.mdfdir is not None:
dirs.append(self.mdfdir)
dname = os.path.dirname(self.filename)
if dname and dname != ".":
dirs.append(dname)
dirs.append(os.getcwd())
# search through semester directories as well
semester_dir = self.get_observatory_prefix() + self.get_semester()
directories_to_search = []
for dname in dirs:
directories_to_search.append(dname)
dname = os.path.join(dname, semester_dir)
directories_to_search.append(dname)
# now search through the directories
for dname in directories_to_search:
fname = os.path.join(dname, mdffile)
debug("...trying", fname)
if os.path.exists(fname):
return fname
raise ValueError("Unable to find MDF file named '%s'" % mdffile)
def get_relates_by_spu_draft_id(spu_draft_id):
result = dict()
sku_draft_ids = get_db_sku_ids(spu_draft_id)
result["spuDraftId"] = spu_draft_id
result["skuDraftIds"] = sku_draft_ids
spu_id = get_db_spu_ids(spu_draft_id, False)
if not (spu_id is None) and bm.is_number(spu_id):
sku_ids = get_db_sku_ids(spu_id, False)
sku_data = mu.get_db_group_info("db_pop.item_sku", "id", sku_ids)
item_ids = list(map(lambda x: x['item_id'], sku_data))
item_ids = list(set(item_ids))
result["spuId"] = spu_id
result["skuIds"] = sku_ids
result["itemIds"] = item_ids
return result
def __init__(self, civicrm, **kwargs):
self.civicrm = civicrm
if 'contact' in kwargs:
contact = kwargs['contact']
elif 'member_id' in kwargs:
contact = civicrm.get('Contact', external_identifier=kwargs['member_id'])[0]
else:
raise ValueError('now loading argument stated')
self.civi_id = int(contact['id'])
if is_number(contact['external_identifier']):
self.member_id = int(contact['external_identifier'])
else:
self.member_id = 0
self.firstname = contact['first_name']
self.lastname = contact['last_name']
self.email = contact['email']
self.country = contact['country']
self.city = contact['city']
self.street = contact['street_address']
self.postalcode = contact['postal_code']
self.phone = contact['phone']
self.state = contact['state_province_name']
if 'memberships' in kwargs:
if kwargs['memberships'] != None:
self.memberships = list();
for membership in kwargs['memberships']:
if membership.contact_id == self.civi_id:
self.memberships.append(membership)
else:
self.memberships = list();
membership_values = civicrm.get('Membership', contact_id=self.civi_id)
for membership_data in membership_values:
membership = Membership(data=membership_data)
self.memberships.append(membership)
if 'verification' in kwargs and kwargs['verification'] == True:
self.verified = False
verification_values = civicrm.get('CustomValue', entity_id=self.civi_id)
for verification_data in verification_values:
if verification_data['id'] == '7' and verification_data['0'] == '1':
self.verified = True
def process_number(self, n_info):
if isinstance(n_info, str) and n_info.lower() == "pi":
return Num(math.pi)
if isinstance(n_info, str) and is_number(n_info):
return Num(float(n_info))
if not isinstance(n_info, tuple):
raise NotImplementedError(
f"[process_number] n_info must be tuple or string")
n_pred = n_info[0].lower()
n_args = n_info[1:]
if n_pred == "dist":
assert (len(n_args) == 2)
p1 = self.process_point(n_args[0])
p2 = self.process_point(n_args[1])
n_val = FuncInfo("dist", [p1, p2])
return Num(n_val)
elif n_pred in ["uangle", "area"]:
assert (len(n_args) == 3)
p1, p2, p3 = [self.process_point(p) for p in n_args]
n_val = FuncInfo(n_pred, [p1, p2, p3])
return Num(n_val)
elif n_pred == "radius":
assert (len(n_args) == 1)
circ = self.process_circle(n_args[0])
n_val = FuncInfo("radius", [circ])
return Num(n_val)
elif n_pred == "diam":
assert (len(n_args) == 1)
circ = self.process_circle(n_args[0])
n_val = FuncInfo("diam", [circ])
return Num(n_val)
elif n_pred in ["div", "add", "sub", "mul", "pow"]:
assert (len(n_args) == 2)
n1, n2 = [self.process_number(n) for n in n_args]
n_val = FuncInfo(n_pred, [n1, n2])
return Num(n_val)
elif n_pred in ["neg", "sqrt"]:
assert (len(n_args) == 1)
n = self.process_number(n_args[0])
n_val = FuncInfo(n_pred, [n])
return Num(n_val)
else:
raise NotImplementedError(
f"[process_number] Unsupporrted number pred: {n_pred}")
def insert_course(db, headings, info):
"""Add a row to the Course table.
Args:
headings (List[str]): headings for the current CSV file.
info (List[str]): [city, category, season].
Returns:
int: The ID of the current course.
"""
city = info[0] if info[0] != 'Stage' else 'Las Vegas'
cat = info[1] if not is_number(info[1]) else 'Stage ' + info[1]
course_id = db.query_file('data/sql/insert_course.sql',
city=city,
cat=cat,
s=info[2]).all()
return course_id[0].course_id
def get_relates_by_spu_draft_id(spu_draft_id):
result = dict()
sku_draft_ids = get_db_sku_ids(spu_draft_id)
result["spuDraftId"] = spu_draft_id
result["skuDraftIds"] = sku_draft_ids
spu_id = get_db_spu_ids(spu_draft_id, False)
if not (spu_id is None) and bm.is_number(spu_id):
sku_ids = get_db_sku_ids(spu_id, False)
sku_data = mu.get_db_group_info("db_pop.item_sku", "id", sku_ids)
item_ids = list(map(lambda x: x['item_id'], sku_data))
item_ids = list(set(item_ids))
result["spuId"] = spu_id
result["skuIds"] = sku_ids
result["itemIds"] = item_ids
return result
def availableCyclicMean(type, stationId, thresholdMinutes):
available, numberOfWeeks, availableCyclicMean = list(), list(), list()
db = sqlite3.connect(db_path_string)
cursor = db.cursor()
if not type in ['bike', 'stand']:
print type
return availableCyclicMean
if type == 'bike':
stationAvailabilityData = cursor.execute(
'SELECT timestamp, availableBikes FROM OldResults WHERE stationId =:stationId',
{"stationId": stationId})
if type == 'stand':
stationAvailabilityData = cursor.execute(
'SELECT timestamp, availableStand FROM OldResults WHERE stationId =:stationId',
{"stationId": stationId})
indices = weekCycleIndices(thresholdMinutes)
for indice in indices:
numberOfWeeks.insert(indice, 0)
available.insert(indice, 0)
for data in stationAvailabilityData:
indice = timestampToWeekCycleIndice(data[0], thresholdMinutes)
if not util.is_number(data[1]):
print 'unknown availability data', data[1]
available[indice] = available[indice] + data[1]
numberOfWeeks[indice] += 1
for indice in indices:
if numberOfWeeks[indice] == 0:
availableCyclicMean.insert(indice, 0)
else:
availableCyclicMean.insert(
indice,
round(float(available[indice]) / numberOfWeeks[indice], 2))
db.close()
return availableCyclicMean
def auto_format(data):
'''data is a single list [ ,,]; return string
'''
ss = []
for x in data:
if util.is_number(x):
if math.fabs(x) < 1:
ss.append(' %.3f ' % x)
elif 1.0 <= math.fabs(x) < 10:
ss.append(' %.2f ' % x)
elif 10 <= math.fabs(x) < 100.:
ss.append(' %.1f ' % x)
elif math.fabs(x) >= 100.:
ss.append(' %.0f ' % (x + 0.0001))
else:
ss.append(' %s ' % x)
return ' '.join(ss)
def create_simplified_projections():
for s in ['fanduel', 'draftkings']:
conf.site = s
conf.rotogrinder_hitter_path =\
'http://rotogrinders.com/projected-stats/mlb-hitter.csv?site=%s' % conf.site
conf.rotogrinder_pitcher_path =\
'http://rotogrinders.com/projected-stats/mlb-pitcher.csv?site=%s' % conf.site
sals_and_pos = retrieve_mlb_player_salaries_and_positions()
if conf.batting_orders is None:
conf.batting_orders, conf.pitchers = retrieve_mlb_batting_order()
df = pd.read_csv('./projections/%s_%s_%s.csv' % (conf.site, conf.proj_date, conf.proj_iteration), index_col=0)
idx = df.index[:]
means = []
for p in idx:
means.append([np.mean(df.loc[p][0:conf.simulated_game_count])])
mean_df = pd.DataFrame(means, index=idx, columns=['mean'])
df = df.join(mean_df)
# join our two data frames
projections_overall = df.join(sals_and_pos, how='left')
# drop players that weren't included
to_drop = []
for name in projections_overall.index:
if type(projections_overall.at[name, 'cost']) == np.ndarray:
print name
print projections_overall.loc[name]
pdb.set_trace()
if not is_number(projections_overall.at[name, 'cost']):
to_drop.append(name)
elif name in conf.excluded_batters or name in\
conf.excluded_pitchers:
to_drop.append(name)
elif conf.use_inclusion is False and projections_overall.at[name, 'team'] in conf.excluded_teams:
to_drop.append(name)
elif conf.use_inclusion is True and projections_overall.at[name, 'team'] not in conf.included_teams:
to_drop.append(name)
projections_overall = projections_overall.drop(to_drop)
projections_overall['exclude'] = 0
projections_overall[['pos', 'mean', 'cost', 'team', 'exclude']].to_csv('./simplified_projections/%s_%s_%s.csv' % (conf.site, conf.proj_date, conf.proj_iteration))
def get_pop_corr(infile, nn, narg, arg):
'''nn, the nth argument; narg, the maximum argument; arg, the argument
'''
xxrange, yyrange, title, xlabel, ylabel, xcol, ycol, nstep = '', '', '', '', '', 0, 0, 0
ss = ''
for i in range(nn, narg):
ss = ss + '%s ' % arg[i]
ss_split = ss.split(',')
for x in ss_split:
s = x.strip().split('=')
key = s[0].strip().lower()
if ('xcol' == key or 'ycol' == key) and not util.is_number(s[1]):
print 'Error: not an integer after xcol or ycol. both must be separated by ",".'
return
if 'xrange' == key:
xxrange = s[1]
elif 'yrange' == key:
yyrange = s[1]
elif 'title' == key:
title = s[1]
elif 'xlabel' == key:
xlabel = s[1]
elif 'ylabel' == key:
ylabel = s[1]
elif 'xcol' == key:
xcol = int(s[1])
elif 'ycol' == key:
ycol = int(s[1])
elif 'nstep' == key:
nstep = int(s[1])
print 'Input file=%s' % infile
if xcol > 0 and ycol > 0: #do correlations
correlation_plot(infile, xcol, ycol, xlabel, ylabel, title, nstep,
xxrange, yyrange)
elif xcol > 0:
histogram_plot(infile, xcol, xlabel, ylabel, title, nstep, xxrange)
def getNearestPrecipitationsForPrevision(times, hourLimit, db_path):
db = sqlite3.connect(db_path)
cursor = db.cursor()
R = dict()
for t in times:
min_time = t-3600*hourLimit
data = cursor.execute('''SELECT timestamp, precipitation FROM WeatherForecast WHERE timestamp>:lowLimit AND timestamp<=:highLimit ORDER BY timestamp DESC''',{"lowLimit":min_time, "highLimit":t}).fetchone()
if data and util.is_number(data[1]):
R[t] = float(data[1])
#print 'found precipitation data', data[1]
#else:
#print 'precipitation data not found'
db.close()
return R
def execute(self):
'''execute the "program" on the code stack'''
while (not self.code.isEmpty()):
#TODO: remove debug
# print "----]] stack: {0} ^^^ {1} ".format(self.data.toString(True), self.code.toString())
# print "----]] stack: {0} ^^^ {1} ".format(self.data.toString(True,80), self.code.toString(False,50))
token = self.code.pop()
if util.is_number(token):
# integers are simply pushed onto the data stack
self.data.push(int(token))
elif token in self.binaryOps:
# binary operators: {+,-,*,/,%,&,|,=,<,>}
# pop to items from data stack
a = self.data.pop()
b = self.data.pop()
#TODO: remove debug
# print "bin op={0}, a={1}, b={2}".format(token,a,b)
# get the operator
op_func = self.binaryOps[token]
if(token != "="): # "=" does not require a and b to be integers
#check that they are integers
util.expectInt(a)
util.expectInt(b)
# apply the operator...
c = int(op_func(a, b)) # some operators (i.e., "<" and ">" return "True" or "False"; so parse it to int
# and push the result onto the data stack
self.data.push(c)
elif token == "~":
# negation
val = int(self.data.pop())
self.data.push(-val)
elif token == "x":
return #quick fix: exit gracefully
elif token in self.commands:
func = self.commands[token]
func()
elif token.startswith("[") and token.endswith("]"):
self.data.push(token)
else:
msg = "Unknown command: " + token
raise ValueError(msg)
def process_circle(self, c_info, unnamed=True):
if isinstance(c_info, str) and not is_number(c_info):
assert (Circle(c_info) in self.circles)
return Circle(c_info)
if not isinstance(c_info, tuple):
raise NotImplementedError(
f"[process_circle] c_info must be tuple or string")
c_pred = c_info[0].lower()
ps = [self.process_point(p) for p in c_info[1:]]
c_val = None
if c_pred == "circ":
assert (len(ps) == 3)
c_val = FuncInfo("c3", ps)
elif c_pred == "coa":
assert (len(ps) == 2)
c_val = FuncInfo(c_pred, ps)
elif c_pred == "diam":
assert (len(ps) == 2)
c_val = FuncInfo("diam", ps)
elif c_pred == "circumcircle":
assert (len(ps) == 3)
c_val = FuncInfo("circumcircle", ps)
elif c_pred == "incircle":
assert (len(ps) == 3)
c_val = FuncInfo("incircle", ps)
elif c_pred == "excircle":
assert (len(ps) == 3)
c_val = FuncInfo("excircle", ps)
elif c_pred == "mixtilinear-incircle":
assert (len(ps) == 3)
c_val = FuncInfo(c_pred, ps)
if c_val is not None:
C = Circle(c_val)
if unnamed:
self.unnamed_circles.append(C)
return C
else:
raise NotImplementedError(
f"[process_circle] Unsupported circle pred: {c_pred}")
def get_tracform_state(self, src, db=None):
db = self._get_db(db)
cursor = db.cursor()
sql = """
SELECT state
FROM forms
"""
if not is_number(src):
sql += """
WHERE realm=%s
AND resource_id=%s
AND subcontext=%s
"""
else:
sql += """
WHERE id=%s
"""
src = tuple([src],)
cursor.execute(sql, src)
row = cursor.fetchone()
return row and row[0]
def get_tracform_state(self, src, db=None):
db = self._get_db(db)
cursor = db.cursor()
sql = """
SELECT state
FROM forms
"""
if not is_number(src):
sql += """
WHERE realm=%s
AND resource_id=%s
AND subcontext=%s
"""
else:
sql += """
WHERE id=%s
"""
src = tuple([src], )
cursor.execute(sql, src)
row = cursor.fetchone()
return row and row[0]
def describe_instances(self, instance_ids):
print "========= A potential variation for probing the region =========="
d = defaultdict(list)
for instance_id in instance_ids:
assert (instance_id < len(self.rows))
row = self.rows[instance_id]
for feature in self.feature_names:
value = row[feature]
if is_number(value):
value = float(value)
if float(value) == -1.0:
continue
feature = remove_XXX(feature)
d[feature].append(value)
for feature in self.feature_names:
feature = remove_XXX(feature)
values = d[feature]
if values and type(values[0]) is float:
lo, hi = min(values), max(values)
if lo == hi:
continue # no variation
elif lo == 0 and hi == 1:
print " * {0:<13}: [{1:}, {2:}]".format(
feature, "no", "yes")
else:
unit = get_unit(feature)
print " * {0:<13}: [{1:}, {2:}] {3:}".format(
feature, lo, hi, unit)
elif values and type(values[0]) is str:
values = list(set(values))
for i in range(0, len(values), 3):
vals = ", ".join(values[i:i + 3])
if i == 0:
print " * {0:<13}: {1:}".format(feature, vals)
else:
print " * {0:<13} {1:}".format("", vals)
def availableCyclicMean(type, stationId, thresholdMinutes):
available, numberOfWeeks, availableCyclicMean = list(), list(), list()
db = sqlite3.connect(db_path_string)
cursor = db.cursor()
if not type in ['bike','stand'] :
print type
return availableCyclicMean
if type == 'bike':
stationAvailabilityData = cursor.execute('SELECT timestamp, availableBikes FROM OldResults WHERE stationId =:stationId', { "stationId": stationId})
if type == 'stand':
stationAvailabilityData = cursor.execute('SELECT timestamp, availableStand FROM OldResults WHERE stationId =:stationId', { "stationId": stationId})
indices = weekCycleIndices(thresholdMinutes)
for indice in indices:
numberOfWeeks.insert(indice, 0)
available.insert(indice, 0)
for data in stationAvailabilityData:
indice = timestampToWeekCycleIndice(data[0], thresholdMinutes)
if not util.is_number(data[1]):
print 'unknown availability data', data[1]
available[indice] = available[indice] + data[1]
numberOfWeeks[indice] += 1
for indice in indices:
if numberOfWeeks[indice] == 0:
availableCyclicMean.insert(indice,0)
else:
availableCyclicMean.insert(indice, round(float(available[indice])/numberOfWeeks[indice],2))
db.close()
return availableCyclicMean
def find_xyzlim_compound(compid, coord):
'''find xyzlimit used by mapmask, and write the coord in cif or pdb format.
compid: atom_group_id (model_compound_chainID_resnumber_alter_insertion)
coord: the coordinate file
idd = 0, cif format; =1, the pdb format
'''
comp = 'XXXX'
t1 = compid.split(':')
for i, x in enumerate(t1):
t = x.split('_')
if i == 0: comp = '_'.join([t[0], t[1], t[2], t[3]])
if len(t) != 6:
print(
'Error: in group-id (%d). it should be (model_compound_chainID_resnumber_alter_insertion).'
% (i + 1))
return '', ''
idd = util.is_cif(coord)
xyzcomp = comp + '.pdb'
if idd == 1: xyzcomp = comp + '.cif'
fw = open(xyzcomp, 'w')
border = 1 #extend a little to cover more density
xx, yy, zz = [], [], []
if idd == 1: #input cif format
fw.write('data_xyzcomp\n#\n')
flist = open(coord, 'r').readlines()
items, values = cif.cifparse(flist, '_cell.')
fw.write('\n#\n')
for m, p in enumerate(items):
fw.write("%s %s\n" % (p, values[m]))
cell = cif.get_cell(flist)
items, values = cif.cifparse(flist, '_atom_site.')
comp = cif.parse_values(items, values, "_atom_site.auth_comp_id")
asym = cif.parse_values(items, values, "_atom_site.auth_asym_id")
seq = cif.parse_values(items, values, "_atom_site.auth_seq_id")
alt = cif.parse_values(items, values, "_atom_site.label_alt_id")
ins = cif.parse_values(items, values, "_atom_site.pdbx_PDB_ins_code")
x = cif.parse_values(items, values, "_atom_site.Cartn_x")
y = cif.parse_values(items, values, "_atom_site.Cartn_y")
z = cif.parse_values(items, values, "_atom_site.Cartn_z")
model = cif.parse_values(items, values,
"_atom_site.pdbx_PDB_model_num")
if (not (alt and comp and ins and asym and seq and x and y and z)):
print(
'Error: not enough infor. extraced from (%s). Check ciftokens'
% coord)
sys.exit()
fw.write('\n#\nloop_\n')
for p in items:
fw.write("%s\n" % p)
row = cif.get_rows(items, values)
for i in range(len(x)):
alter, inst, mod = '.', '.', '1'
if model and util.is_number(model[i]): mod = model[i]
if alt and alt[i] != '?': alter = alt[i]
if ins and ins[i] != '?': inst = ins[i]
id1 = '_'.join([mod, comp[i], asym[i], seq[i], alter, inst])
if id1 in compid:
xx.append(float(x[i]))
yy.append(float(y[i]))
zz.append(float(z[i]))
for m in row[i]:
fw.write("%s " % m)
fw.write('\n')
else: #pdb format
fp = open(coord, 'r')
for x1 in fp:
if ('CRYST1' in x1[:6]):
fw.write(x1)
cell = [float(p) for p in x1[8:54].split()]
elif ('ATOM' in x1[:4] or 'HETATM' in x1[:6]):
alt = x1[16:17]
if alt.isspace(): alt = '.'
ins = x1[26:27]
if ins.isspace(): ins = '.'
resname, chid, resnum = x1[17:20].strip(), x1[20:22].strip(
), x1[22:26].strip()
resid = '_'.join([resname, chid, resnum, alt, ins])
if resid in compid:
fw.write(x1) #only write the selected section
xx.append(float(x1[30:38]))
yy.append(float(x1[38:46]))
zz.append(float(x1[46:54]))
fp.close()
if not xx or not yy or not zz:
print('Error: %s can not be found in the coordinate. try a new id. ' %
(compid))
return '', ''
frac, orth = util.frac_orth_matrix(cell) #get matrix
border = 2.0
xx_min, xx_max = min(xx) - border, max(xx) + border
yy_min, yy_max = min(yy) - border, max(yy) + border
zz_min, zz_max = min(zz) - border, max(zz) + border
xf_min = util.matrix_prod(frac, [xx_min, yy_min, zz_min])
xf_max = util.matrix_prod(frac, [xx_max, yy_max, zz_max])
xyzlim = '%.3f %.3f %.3f %.3f %.3f %.3f' % (
xf_min[0], xf_max[0], xf_min[1], xf_max[1], xf_min[2], xf_max[2])
fw.close()
return xyzlim, xyzcomp
def model_vs_data_log(infile,dic1, dic):
'''extract values and put them into dic (should be often updated)
'''
if not util.check_file(100, infile) : return
dic['prog_c'] = 'PHENIX'
fr=open(infile, 'r')
for ln in fr:
if 'Stereochemistry' in ln and 'overall:' in ln:
for y in fr:
t=y.split(':')[1].split()
if 'bonds :' in y:
dic['bond'], dic['bond_max'] = t[0], t[1]
elif 'angles :' in y:
dic['angle'], dic['angle_max'] = t[0], t[1]
elif 'dihedrals :' in y:
dic['dihedral'], dic['dihedral_max'] = t[0], t[1]
elif 'chirality :' in y:
dic['chirality'], dic['chirality_max'] = t[0], t[1]
elif 'planarity :' in y:
dic['planarity'], dic['planarity_max'] = t[0], t[1]
elif 'non-bonded (min) :' in y:
dic['non_bond'] = t[0].strip()
break
elif 'Stereochemistry' in ln and 'ligands:' in ln:
for y in fr:
t=y.split(':')[1].split()
if 'bonds :' in y:
dic['bond_lig'], dic['bond_lig_max'] = t[0], t[1]
elif 'angles :' in y:
dic['angle_lig'], dic['angle_lig_max'] = t[0], t[1]
elif 'non-bonded (min) :' in y:
break
elif 'Molprobity statistics:' in ln:
for y in fr:
if 'outliers :' in y:
dic['outlier_num'] = util.str_between_id(y,':', '(')
dic['outlier'] = util.str_between_id(y,'(', '%')
elif ' allowed :' in y:
dic['allowed_num'] = util.str_between_id(y,':', '(')
dic['allowed'] = util.str_between_id(y,'(', '%')
elif ' favored :' in y:
dic['favored_num'] = util.str_between_id(y,':', '(')
dic['favored'] = util.str_between_id(y,'(', '%')
elif ' Rotamer outliers' in y:
dic['rot_outlier_num'] = util.str_between_id(y,':', '(')
dic['rot_outlier'] = util.str_between_id(y,'(', '%')
elif 'Cbeta deviations' in y:
dic['cbeta'] = y.strip().split(':')[1].split()[0]
elif 'All-atom clashscore ' in y:
dic['clashscore'] = y.strip().split(':')[1].split()[0]
elif 'Overall score ' in y:
dic['oscore'] = y.strip().split(':')[1].split()[0]
break
elif 'mFo-DFc map:' in ln:
for y in fr:
if ' > 3 sigma:' in y :
dic['fofc_3sig_p'] = y.strip().split(':')[1]
elif ' > 6 sigma:' in y :
dic['fofc_6sig_p'] = y.strip().split(':')[1]
elif '< -3 sigma:' in y:
dic['fofc_3sig_n'] = y.strip().split(':')[1]
elif '< -6 sigma:' in y:
dic['fofc_6sig_n'] = y.strip().split(':')[1]
break
elif 'bulk_solvent_(k_sol,b_sol' in ln:
tmp=ln.split(':')[1].strip().split()
dic1['k_sol'] = tmp[0]
dic1['b_sol'] = tmp[1]
elif 'high_resolution :' in ln or 'high_resolution :' in ln :
dic1['resh']=ln.split(':')[1].strip()
elif 'low_resolution :' in ln or 'low_resolution :' in ln :
dic1['resl']=ln.split(':')[1].strip()
elif 'completeness_in_range' in ln and ':' in ln:
tmp=ln.split(':')[1].strip()
if(float(tmp)<1.1):
dic1['comp']="%.3f" %(float(tmp)*100)
else:
dic1['comp']=tmp
elif 'wilson_b :' in ln or 'wilson_b :' in ln :
dic1['bwilson']=ln.split(':')[1].strip()
elif ' number_of_reflections :' in ln or 'number_of_reflections :' in ln :
dic1['nref']=ln.split(':')[1].strip()
elif 'test_set_size :' in ln or 'test_set_size :' in ln:
tmp=ln.split(':')[1].strip()
if('?' not in dic1['nref']) :
dic1['nfree']="%d" %(float(tmp)*float(dic1['nref']))
elif 'twinned :' in ln:
twin=ln.split(':')[1].strip()
if 'False' not in twin : dic1['twin'] = 'Y'
elif 'r_work(re-computed) :' in ln:
dic1['rfact']=ln.split(':')[1].strip()
elif 'r_free(re-computed) :' in ln:
t=ln.split(':')[1].strip()
if not util.is_number(t) : t='?'
dic1['rfree']=t
elif 'TLS : ' in ln:
tls_rep=ln.split(':')[1].strip()
if 'False' not in tls_rep : dic1['tls_c'] = 'Y'
elif 'program_name :' in ln:
dic1['prog_r']=ln.split(':')[1].strip()
elif 'r_work :' in ln:
dic1['rfact_r']=ln.split(':')[1].strip()
elif 'r_free :' in ln:
dic1['rfree_r']=ln.split(':')[1].strip()
elif 'high_resolution :' in ln:
dic1['resh_r']=ln.split(':')[1].strip()
elif 'low_resolution :' in ln:
dic1['resl_r']=ln.split(':')[1].strip()
elif 'r_work_cutoff :' in ln:
dic1['r_cutoff']=util.float_after_id(ln, ':')
elif 'r_free_cutoff :' in ln:
dic1['rf_cutoff']=util.float_after_id(ln, ':')
elif 'PDB_string' in ln: #for density correlation(atom list)
dic1['dcc'] = get_dcc_avg(fr, 1)
elif 'resseq resname' in ln: #for density correlation(residue list)
dic1['dcc'] = get_dcc_avg(fr, 2)
# print(dic['dcc'])
fr.close()
def encode(num):
if not is_number(num) or num < 0:
raise Exception("num is not numeric or < 0!")
return _10baseN(num, len(_URL_SAFE_SET), _URL_SAFE_SET)
def update_totalOzone_master_file(self, directory, master_file, date, mode,
heading): # noqa
"""Updates Total Ozone Master File"""
# Initialization
write_output = 1
current_time = (datetime.now()).strftime("%Y_%m_%d")
log_file = open('totalOzone_processing_log_%s' % current_time,
'wb') # noqa
data_file = None
global tmp_filename
tmp_filename = os.path.join(master_file, 'o3tot')
if mode == 'overwrite':
data_file = open(tmp_filename, 'wb+')
else:
data_file = open(tmp_filename, 'ab+')
if heading == 'on':
data_file.write(
'Platform_ID,Year,Month,Day,Start_Hour,Finish_Hour,Wavelength_Pair,Observation_Type,Total_Column_Ozone_Amount,Ozone_Std_Error,Instrument_Type,Instrument_Number\r\n'
) # noqa
# external ftp file
global output_file
output_file = 'Summaries/TotalOzone/Daily_Summary/o3tot.zip' # noqa
# extract zipfile
zip_flag = False
path = directory
if zipfile.is_zipfile(directory):
zip_flag = True
tmpdir = tempfile.mkdtemp()
z = zipfile.ZipFile(directory)
z.extractall(path=tmpdir)
path = tmpdir
# traverse the given directory
for dirname, dirnames, filenames in os.walk(path):
dirnames.sort()
filenames.sort()
for filename in filenames:
try:
# print filename
file_last_modified_date = time.strftime(
"%Y-%m-%d",
time.localtime(
os.path.getmtime(os.path.join(dirname,
filename)))) # noqa
# date comparison
if date is not None and file_last_modified_date <= date: # noqa
log_file.write(
'PROCESSED#%s last modified date: %s\r\n' %
((os.path.join(dirname, filename)),
file_last_modified_date)) # noqa
extCSV = WOUDCextCSVReader(
os.path.join(dirname, filename)) # noqa
if date is not None and file_last_modified_date > date:
continue
if date is None:
log_file.write(
'PROCESSED#%s last modified date: %s\r\n' %
((os.path.join(dirname, filename)),
file_last_modified_date)) # noqa
extCSV = WOUDCextCSVReader(
os.path.join(dirname, filename)) # noqa
# store data into variables
platform_id = ' '
if 'PLATFORM' in extCSV.sections:
p_id = extCSV.sections['PLATFORM']['ID']
if p_id is not None and len(p_id) != 0:
platform_id = p_id
if len(platform_id) == 1:
platform_id = '00%s' % platform_id
if len(platform_id) == 2:
platform_id = '0%s' % platform_id
else:
log_file.write(
'ERROR#E01:Could not find PLATFORM in input file: %s. Data is ignored\r\n'
% os.path.join(dirname, filename)) # noqa
inst_type_id = ' '
inst_number = ' 0'
if 'INSTRUMENT' in extCSV.sections:
inst_name = extCSV.sections['INSTRUMENT']['Name']
inst_model = extCSV.sections['INSTRUMENT']['Model']
if inst_name is not None and len(
inst_name) != 0 and inst_name: # noqa
try:
inst_type_id = util.get_config_value(
'Instrument Type ID', inst_name) # noqa
if inst_model == 'Japanese':
inst_type_id = util.get_config_value(
'Instrument Type ID',
inst_model + ' ' + inst_name) # noqa
if len(inst_type_id) == 1:
inst_type_id = ' %s' % inst_type_id
except Exception, err:
log_file.write(
'ERROR#E02:There is no instrumet type id for \'%s\' in file %s. Data is ignored\r\n'
%
(inst_name, os.path.join(
dirname, filename))) # noqa
write_output = 0
pass
i_num = extCSV.sections['INSTRUMENT']['Number']
if i_num is not None and len(i_num) != 0:
inst_number = i_num
inst_number = re.sub("^0{1,2}", "",
inst_number) # noqa
if len(inst_number) == 1:
inst_number = ' %s' % inst_number
if len(inst_number) == 2:
inst_number = ' %s' % inst_number
if len(inst_number) == 3:
inst_number = ' %s' % inst_number
if i_num == 'na':
inst_number = ' 0'
else:
log_file.write(
'ERROR#E03:Could not find INSTRUMENT in input file: %s. Data is ignored\r\n'
% os.path.join(dirname, filename)) # noqa
if 'DAILY' in extCSV.sections:
data = StringIO(
(extCSV.sections['DAILY']['_raw']).strip()) # noqa
if data is not None:
try:
data_rows = csv.reader(data)
data_rows.next()
except StopIteration:
log_file.write(
'ERROR#E04:Error reading DAILY block in file %s. Data is ignored\r\n'
% os.path.join(dirname, filename)) # noqa
write_output = 0
pass
for row in data_rows:
year = ' '
month = ' '
day = ' '
UTC_Begin = ' '
UTC_End = ' '
WLCode = ' '
ObsCode = ' '
ozone_std_error = ' '
ColumnO3 = ' '
UTC_Mean = ' '
nObs = ' '
if len(row) > 1 and "*" not in row[0]:
if len(row[0]) != 0:
year = row[0].split('-')[0]
month = row[0].split('-')[1]
day = row[0].split('-')[2]
if len(row) >= 2:
if len(row[1]) != 0:
WLCode = row[1]
if len(WLCode) > 1:
try:
WLCode = util.get_config_value(
'WLCode',
WLCode) # noqa
except Exception, err:
log_file.write(
'ERROR#E05:There is no one character WLCode code for \'%s\' in file %s. Data is ignored\r\n'
%
(WLCode,
os.path.join(
dirname, filename)
)) # noqa
write_output = 0
pass
else:
if inst_name == 'Dobson':
WLCode = util.get_config_value(
'WLCode', 'Dobson') # noqa
if inst_name == 'Brewer':
WLCode = util.get_config_value(
'WLCode', 'Brewer') # noqa
if inst_name == 'Filter':
WLCode = util.get_config_value(
'WLCode', 'Filter') # noqa
if inst_name == 'Microtops':
WLCode = util.get_config_value(
'WLCode',
'Microtops') # noqa
if len(row) >= 3:
if len(row[2]) != 0:
ObsCode = row[2]
if util.is_number(
ObsCode) == False and len(
ObsCode) != 1: # noqa
try:
ObsCode = util.get_config_value(
'Obs Code',
ObsCode) # noqa
except Exception, err:
log_file.write(
'ERROR#E06:There is no obs code for \'%s\' in file %s. Data is ignored\r\n'
%
(ObsCode,
os.path.join(
dirname, filename)
)) # noqa
write_output = 0
pass
else:
ObsCode = '9'
if len(row) >= 4:
if len(
row[3]
) != 0 and row[3] != '0.0' and row[
3] != '0' and not "-" in row[
3]: # noqa
try:
ColumnO3 = '%.0f' % round(
float(
re.findall(
"[0-9]*.[0-9]*",
row[3])[0]),
0) # noqa
if ColumnO3 == '0':
write_output = 0
except Exception, err:
log_file.write(
'ERROR#E07:Could not round ColumnO3 value of: %s in file %s. Data ignored.\r\n'
% (ColumnO3,
os.path.join(
dirname,
filename))) # noqa
write_output = 0
if len(ColumnO3) == 1:
ColumnO3 = ' %s' % ColumnO3 # noqa
if len(ColumnO3) == 2:
ColumnO3 = ' %s' % ColumnO3 # noqa
else:
write_output = 0
if len(row) >= 6:
if len(row[5]) != 0:
UTC_Begin = row[5]
if len(
re.findall(
"[0-9]*", UTC_Begin)
[0]) > 2: # noqa
UTC_Begin = UTC_Begin[:2]
elif "-" in UTC_Begin:
if -1.5 >= float(
UTC_Begin): # noqa
UTC_Begin = '-0'
else:
UTC_Begin = '00'
else:
try:
UTC_Begin = '%.0f' % round(
float(UTC_Begin),
0) # noqa
except Exception, err:
log_file.write(
'ERROR#E08:Could not round UTC_Begin value of: %s in file %s. Data ignored.\r\n'
%
(UTC_Begin,
os.path.join(
dirname, filename)
)) # noqa
write_output = 0
if int(UTC_Begin) in range(
10): # noqa
UTC_Begin = '0%s' % UTC_Begin # noqa
if len(row) >= 7:
if len(row[6]) != 0:
UTC_End = row[6]
if len(
re.findall(
"[0-9]*", UTC_End)
[0]) > 2: # noqa
UTC_End = UTC_End[:2]
elif "-" in UTC_End:
if -1.5 >= float(UTC_End):
UTC_End = '-0'
else:
UTC_End = '00'
else:
try:
UTC_End = '%.0f' % round(
float(UTC_End),
0) # noqa
except Exception, err:
log_file.write(
'ERROR#E09:Could not round UTC_End value of: %s in file %s. Data ignored.\r\n'
%
(UTC_End,
os.path.join(
dirname, filename)
)) # noqa
write_output = 0
if int(UTC_End) in range(
10): # noqa
UTC_End = '0%s' % UTC_End # noqa
if len(row) >= 8:
if len(row[7]) != 0:
UTC_Mean = row[7]
if len(row[6]) == 0:
UTC_End = UTC_Mean
if "-" in UTC_End:
if float(
UTC_End
) <= -1.5 and float(
UTC_End
) > -2: # noqa
UTC_End = '-0'
elif float(
UTC_End
) <= -1 and float(
UTC_End
) > -2: # noqa
UTC_End = '00'
elif float(UTC_End
) >= -1: # noqa
UTC_End = '00'
elif float(
UTC_End
) <= -2 and float(
UTC_End
) >= -10: # noqa
UTC_End = '-0'
elif float(
UTC_End
) < -10 and float(
UTC_End
) >= -10.5: # noqa
UTC_End = '-0'
else:
UTC_End = '-1'
else:
try:
UTC_End = '%.0f' % round(
float(UTC_End),
0) # noqa
except Exception, err:
log_file.write(
'ERROR#E09:Could not round UTC_End value of: %s in file %s. Data ignored.\r\n'
% (UTC_End,
os.path.join(
dirname,
filename))
) # noqa
write_output = 0
if int(UTC_End) in range(
10): # noqa
UTC_End = '0%s' % UTC_End # noqa
def process_point(self, p_info, unnamed=True):
if isinstance(p_info, str) and not is_number(p_info):
assert (Point(p_info) in self.points)
return Point(p_info)
if not isinstance(p_info, tuple):
raise NotImplementedError(
f"[process_point] p_info must be tuple or string")
p_pred = p_info[0].lower()
p_args = p_info[1:]
p_val = None
if p_pred == "inter-ll":
assert (len(p_args) == 2)
l1 = self.process_line(p_args[0])
l2 = self.process_line(p_args[1])
p_val = FuncInfo(p_pred, (l1, l2))
elif p_pred in ["isogonal-conj", "isotomic-conj"]:
assert (len(p_args) == 4)
ps = [self.process_point(p) for p in p_args]
p_val = FuncInfo(p_pred, tuple(ps))
elif p_pred == "harmonic-conj":
assert (len(p_args) == 3)
ps = [self.process_point(p) for p in p_args]
p_val = FuncInfo(p_pred, tuple(ps))
elif p_pred in ["incenter", "excenter", "mixtilinear-incenter"]:
assert (len(p_args) == 3)
ps = [self.process_point(p) for p in p_args]
p_val = FuncInfo(p_pred, tuple(ps))
elif p_pred == "inter-lc":
assert (len(p_args) == 3)
l = self.process_line(p_args[0])
c = self.process_circle(p_args[1])
rs = self.process_rs(p_args[2])
p_val = FuncInfo(p_pred, (l, c, rs))
elif p_pred == "inter-cc":
assert (len(p_args) == 3)
c1 = self.process_circle(p_args[0])
c2 = self.process_circle(p_args[1])
rs = self.process_rs(p_args[2])
p_val = FuncInfo(p_pred, (c1, c2, rs))
elif p_pred in ["midp", "midp-from"]:
assert (len(p_args) == 2)
ps = [self.process_point(p) for p in p_args]
p_val = FuncInfo(p_pred, tuple(ps))
elif p_pred == "foot":
assert (len(p_args) == 2)
p = self.process_point(p_args[0])
l = self.process_line(p_args[1])
p_val = FuncInfo(p_pred, (p, l))
elif p_pred == "reflect-pl":
assert (len(p_args) == 2)
p = self.process_point(p_args[0])
l = self.process_line(p_args[1])
p_val = FuncInfo(p_pred, (p, l))
elif p_pred in ["orthocenter", "circumcenter", "centroid", "incenter"]:
assert (len(p_args) == 3)
ps = [self.process_point(p) for p in p_args]
p_val = FuncInfo(p_pred, tuple(ps))
elif p_pred == "origin":
assert (len(p_args) == 1)
circ = self.process_circle(p_args[0])
p_val = FuncInfo(p_pred, (circ, ))
elif p_pred in ["amidp-opp", "amidp-same"]:
assert (len(p_args) == 3)
ps = [self.process_point(p) for p in p_args]
p_val = FuncInfo(p_pred, tuple(ps))
if p_val is not None:
P = Point(p_val)
if unnamed:
self.unnamed_points.append(P)
return P
else:
raise NotImplementedError(
f"[process_point] Unrecognized p_pred {p_pred}")
def main():
# #####
# In Pattern mode, the drone stabilizes above a black pattern that looks like # #
# # #
#
# TX is the wireless camera we were using, and which required special handling:
# In TX mode the image is flipped vertically, a small black area near
# the borders is removed, and simple de-interlacing is applied.
#
txMode = False
#
# DEVO mode is for the older remotes that use a physical Walkera DEVO
# drone controller. Leave this off of the small new remotes.
#
devoMode = False
#
# If we are running on a respberry computer,
# we have the camera interface to grab photos from the RaspberryCam.
#
berryMode = False
#
# Images on RaspberryCam are too big,
# we need to scale them down.
#
scaleFactor = 1.0
# create a quad serial object
serial = quad_serial.QuadSerial()
pid_controllers = [
pid.Pid(0.04, 0, 0),
pid.Pid(0.06, 0, 0),
pid.Pid(0.04, 0, 0),
pid.Pid(0.3, 0, 0)
]
coeffs = {
"pid": {
"xz": {
"p": 4,
"i": 10,
"d": 25
},
"y": {
"p": 5,
"i": 10,
"d": 33
},
"r": {
"p": 30,
"i": 0,
"d": 0
}
},
"smoothing": 40
}
cameraIndex = 0
# handle arguments
for arg in sys.argv[1:]:
if arg == "-ngui":
viz.GUI.viz_mode = False
elif temp == "-pi":
berryMode = True
elif temp == "-nconfig":
viz.GUI.config_mode = False
elif util.is_number(temp):
cameraIndex = int(temp)
viz.GUI.create_trackbars(coeffs)
cap = cv.VideoCapture(cameraIndex)
frame = np.array([])
camera_matrix = PI_CAMERA_MAT
camera_square = np.zeros(3)
drone_translation = np.zeros(3)
drone_rotation = np.zeros(3)
smooth_position = np.zeros(3)
last_frame_tick_count = 0
flight_mode_on = False
paused = False
pressedKey = 0
while pressedKey != 'q':
logging.Debug.debug("************** Entered Loop *****************")
if not paused:
ret, frame = cap.read()
if ret is None:
break
logging.Debug.debug("Photo grabing")
if scaleFactor != 1.0:
resize(frame, frame, Size(), scaleFactor, scaleFactor,
cv.INTER_NEAREST)
logging.Debug.debug("Resizing")
# TX image processing
if txMode:
frame = frame[5:, :-7]
for i in range(0, frame.rows - 1, 2):
frame.row(i + 1).copyTo(frame.row(i))
cv.flip(frame, frame, -1)
logging.Debug.debug("TX Mode")
viz.GUI.copy_frames(frame)
deltaTime = float(cv.getTickCount() -
last_frame_tick_count) / cv.getTickFrequency()
last_frame_tick_count = cv.getTickCount()
logging.Debug.debug("Time since last frame tick count")
camera_square = image_processing.ImageProcessing.find_open_square(
frame)
viz.GUI.draw_square(frame, camera_square)
logging.Debug.debug("Tracking OpenSquare section")
if camera_square is None:
logging.Debug.debug("Square not found")
else:
retval, rvec, tvec = cv.solvePnP(WORLD_SQUARE,
np.float64(camera_square),
camera_matrix, 0)
logging.Debug.debug("SolvePnP")
"""
camera_transform = np.concatenate((rvec, tvec.T), axis=1)
# The square doesn't move, we are moving.
camera_transform = np.linalg.inv(camera_transform)
# We found the camera, but we want to find the drone
# TODO: Nnot working drone_transform = camera_transform * INV_DRONE_CAMERA_TRANSFORM
pos = drone_transform.translation()
"""
pos = tvec
smooth_position = coeffs["smoothing"] * smooth_position + (
1 - coeffs["smoothing"]) * pos
drone_translation = smooth_position
drone_rotationd = rvec
logging.Debug.debug("found Position")
# get latest coeffs from trackbars
coeffs = viz.GUI.get_trackbars(coeffs)
# update controller coefficients based on trackbar
for index, axis in enumerate(["xz", "y", "xz", "r"]):
factor = 1000.0 if (axis != "r") else 100.0
pid_controllers[index]._kp = coeffs["pid"][axis]["p"] / factor
pid_controllers[index]._ki = coeffs["pid"][axis]["i"] / (factor *
10)
pid_controllers[index]._kd = coeffs["pid"][axis]["d"] / (factor *
10)
controlErrors = util.calculate_control_errors(drone_translation,
drone_rotation,
DRONE_TARGET)
logging.Debug.debug("PID")
# set pid controls
pid_control = [0, 0, 0, 0]
if camera_square is not None and flight_mode_on:
for i in range(4):
pid_control.append(pid_controllers[i].calculate(
controlErrors[i], deltaTime))
logging.Debug.debug("Take off and Landing")
channel_controls = channels.controls_to_channels(pid_control)
if not flight_mode_on:
channel_controls = [64, 64, 0, 64]
if serial.is_opened():
success = serial.send([(int(channel_controls[0]) - 64) * 2,
(int(channel_controls[1]) - 64) * 2,
(int(channel_controls[2]) - 64) * 2,
(int(channel_controls[3]) - 64) * 2])
if not success:
logging.Debug.debug("Failed to send to Arduino")
logging.Debug.debug("Control errors to console")
# Draw GUI
viz.GUI.simple_flight_viz(flight_mode_on)
"""
displayedFrame,
displayedFrame,
drone_transform,
59.9,
droneTarget,
pidControl * 100,
flight_mode_on
"""
logging.Debug.debug("Draw GUI")
pressedKey = cv.waitKey(0)
print type(pressedKey)
print pressedKey
logging.Debug.debug("cv.waitKey(1)")
if pressedKey == ' ':
paused = not paused
elif pressedKey in [ENTER, PI_ENTER, 'x']:
flight_mode_on = not flight_mode_on
elif pressedKey in [BACK_BUTTON, PI_BACK_BUTTON, 'i']:
for pid_obj in pid_controllers:
pid_obj._scaled_error_sum = 0
elif pressedKey in ['q']:
break
elif pressedKey >= '0' and pressedKey <= '9':
viz.GUI.gui_step = pressedKey - '0'
logging.Debug.debug("User Commands")
def query_bam(guidesbam, coords, offcoords=True, onebased=False):
"""Query BAM file containing guideRNA database.
Input:
guidesbam: path to BAM file
coords: genomic coordinates, e.g., str 'chrX:3364088-3372035'
or tuple (chr, start, end), e.g., ('chrX', 3364088, 3372035) ;
assume coordinates are 0-based
offcoords: if True, parse off-target coordinates (takes time), otherwise
keep them in int encoded form (can be decoded later
with util.map_int_to_coord());
if parsed, coordinates of off-targets are 0-based
onebased: if True, input coordinates are 1-based (default is 0-based)
Output:
list of instances of class GuideRNA
"""
#create dictionary to verify query coordinates correspond to organism chromosome identity and length
#chromosome_dictionary = organism_chromosome_identity_and_length_confirmation(guidesbam)
if isinstance(coords, basestring):
coords = coords.strip().split(':')
coords = (coords[0], ) + tuple(int(x) for x in coords[1].split('-'))
chrom, start, end = coords
#confirm coordinate exists in genome
"""
if chromosome_dictionary.has_key(chrom):
if start <= chromosome_dictionary[chrom] and end <= chromosome_dictionary[chrom]:
pass
else:
sys.stderr.write('WARNING: %s and/or %s fall off %s which has length %s \n' % (start,end,chrom,chromosome_dictionary[chrom]))
return
else:
sys.stderr.write('WARNING: %s is not a recognized chromosome for this organism \n' % (chrom))
return
"""
if onebased:
start -= 1 # switch to 0-based coordinates, as expected by pysam
end -= 1 # and used internally in our pipeline (in the trie etc.)
bam = pysam.AlignmentFile(guidesbam, "rb")
guides = []
argsdict = ast.literal_eval(bam.header['CO'][3])
genome = argsdict['genome']
delim = util.get_nonexist_int_coord(genome)
for r in bam.fetch(chrom, start, end):
seq = r.qname
positions = r.get_reference_positions()
# note: pysam returns 0-based coordinates
r_start, r_end = min(positions), max(positions)
if r_start < start or r_end > end:
# ignore guideRNAs not completely within query genomic region
continue
strand = '+' if not r.is_reverse else '-'
tags = dict(r.get_tags())
offtargets = tags.get('of', [])
offdist = tags.get('od', -1)
# if 'ds' in r.tags[len(r.tags)-1]:
# score = r.tags[len(r.tags)-1][1]
score = tags.get('ds', -1)
try:
if util.is_number(score):
score = float(score)
else:
score = -1
except ValueError:
sys.stderr.write(
'ERROR: score %s could not be converted to float, skipping region'
% (score))
return
if offdist == -1:
offdist = argsdict.get('offdist', -1)
maxoffcount = tags.get('oc', -1)
if maxoffcount == -1:
maxoffcount = argsdict.get('maxoffcount', -1)
if isinstance(offtargets, basestring):
offtargets = util.hex_to_offtargetinfo(offtargets, delim,
offcoords)
offtargets = [
(p[0], (1 if p[1] == 0 else p[1]) if not offcoords else
(1 if p[1][0] == 0 else p[1][0]), [] if not offcoords else [
util.map_int_to_coord(
x, genome, onebased=False, strout=False)
for x in p[1][1:]
]) for p in offtargets
]
guiderna = GuideRNA(seq=seq,
coord=(chrom, r_start, r_end, strand),
score=score,
offdist=offdist,
maxoffcount=maxoffcount,
offtargets=offtargets)
guides.append(guiderna)
bam.close()
return guides
def reset_tracform(self, src, field=None, author=None, step=0, db=None):
"""Delete method for all TracForms db tables.
Note, that we only delete recorded values and history here, while
the form definition (being part of forms parent resource) is retained.
Reset of single fields is not implemented, because this would require
cumbersome and error prown history rewriting - not worth the hassle.
"""
db = self._get_db(db)
cursor = db.cursor()
form_ids = []
# identify form_id(s) to reset
if is_number(src):
form_ids.append(src)
elif isinstance(src, tuple) and len(src) == 3:
if src[-1] is None:
# no subcontext given, reset all forms of the parent resource
for form_id in self.get_tracform_ids(src[0], src[1], db=db):
form_ids.append(form_id)
else:
form_ids.append(self.get_tracform_meta(src, db=db)[0])
db = self._get_db(db)
cursor = db.cursor()
# restore of old values for multiple forms is not meaningful
if step == -1 and len(form_ids) == 1:
form_id = form_ids[0]
now = int(time.time())
author, updated_on, old_state = self.get_tracform_history(
form_id, db=db)[0] or \
(author, now, '{}')
if updated_on == now:
# no history recorded, so only form values can be reset
step = 0
else:
# copy last old_state to current
cursor.execute(
"""
UPDATE forms
SET author=%s,
time=%s,
state=%s
WHERE id=%s
""", (author, updated_on, old_state, form_id))
history = []
records = self.get_tracform_history(form_id, db=db)
for history_author, history_time, old_state in records:
history.append({
'author': history_author,
'time': history_time,
'old_state': old_state
})
history = parse_history(history, fieldwise=True)
# delete restored history entry
cursor.execute(
"""
DELETE
FROM forms_history
WHERE id=%s
AND time=%s
""", (form_id, updated_on))
# rollback field info changes
for field in history.keys():
changes = history[field]
if len(changes) > 0:
# restore last field info, unconditional by intention
# i.e. to not create entries, if track_fields is False
cursor.execute(
"""
UPDATE forms_fields
SET author=%s,
time=%s
WHERE id=%s
AND field=%s
""", (changes[0]['author'], changes[0]['time'],
form_id, field))
else:
# delete current field info
cursor.execute(
"""
DELETE
FROM forms_fields
WHERE id=%s
AND field=%s
""", (form_id, field))
if step == 0:
# reset all fields and delete full history
for form_id in form_ids:
cursor.execute(
"""
DELETE
FROM forms_history
WHERE id=%s
""", (form_id, ))
cursor.execute(
"""
DELETE
FROM forms_fields
WHERE id=%s
""", (form_id, ))
# don't delete basic form reference but save the reset
# as a form change to prevent creation of a new form_id
# for further retention data
cursor.execute(
"""
UPDATE forms
SET author=%s,
time=%s,
state=%s
WHERE id=%s
""", (author, int(time.time()), '{}', form_id))
db.commit()
break
return 9999
while True:
offer = telega.last_offer()
txt = offer.message
bet = txt.split("\n")[-1]
if offer.id != last_msg_id:
bf = open('bids')
for i in bf.read().split("\n"):
first_s = i.find(' ')
if first_s < 1 or len(
i[first_s + 1:]) == 0 or not util.is_number(i[:first_s]):
if len(i) > 0:
print("Wrong format for bid: " + i)
continue
if i[first_s + 1:] in txt and init_cost(i) <= int(i[:first_s]):
sleep(random.randrange(1, 15))
telega.send_command(bet + "_" + i[:first_s])
bf.close()
if last_msg_id != offer.id:
print("New offer: " + bet + "; " + txt.split("\n")[1])
last_msg_id = offer.id
sleep(1)
def reset_tracform(self, src, field=None, author=None, step=0, db=None):
"""Delete method for all TracForms db tables.
Note, that we only delete recorded values and history here, while
the form definition (being part of forms parent resource) is retained.
Reset of single fields is not implemented, because this would require
cumbersome and error prown history rewriting - not worth the hassle.
"""
db = self._get_db(db)
cursor = db.cursor()
form_ids = []
# identify form_id(s) to reset
if is_number(src):
form_ids.append(src)
elif isinstance(src, tuple) and len(src) == 3:
if src[-1] is None:
# no subcontext given, reset all forms of the parent resource
for form_id in self.get_tracform_ids(src[0], src[1], db=db):
form_ids.append(form_id)
else:
form_ids.append(self.get_tracform_meta(src, db=db)[0])
db = self._get_db(db)
cursor = db.cursor()
# restore of old values for multiple forms is not meaningful
if step == -1 and len(form_ids) == 1:
form_id = form_ids[0]
now = int(time.time())
author, updated_on, old_state = self.get_tracform_history(
form_id, db=db)[0] or \
(author, now, '{}')
if updated_on == now:
# no history recorded, so only form values can be reset
step = 0
else:
# copy last old_state to current
cursor.execute("""
UPDATE forms
SET author=%s,
time=%s,
state=%s
WHERE id=%s
""", (author, updated_on, old_state, form_id))
history = []
records = self.get_tracform_history(form_id, db=db)
for history_author, history_time, old_state in records:
history.append({'author': history_author,
'time': history_time,
'old_state': old_state})
history = parse_history(history, fieldwise=True)
# delete restored history entry
cursor.execute("""
DELETE
FROM forms_history
WHERE id=%s
AND time=%s
""", (form_id, updated_on))
# rollback field info changes
for field in history.keys():
changes = history[field]
if len(changes) > 0:
# restore last field info, unconditional by intention
# i.e. to not create entries, if track_fields is False
cursor.execute("""
UPDATE forms_fields
SET author=%s,
time=%s
WHERE id=%s
AND field=%s
""", (changes[0]['author'], changes[0]['time'],
form_id, field))
else:
# delete current field info
cursor.execute("""
DELETE
FROM forms_fields
WHERE id=%s
AND field=%s
""", (form_id, field))
if step == 0:
# reset all fields and delete full history
for form_id in form_ids:
cursor.execute("""
DELETE
FROM forms_history
WHERE id=%s
""", (form_id,))
cursor.execute("""
DELETE
FROM forms_fields
WHERE id=%s
""", (form_id,))
# don't delete basic form reference but save the reset
# as a form change to prevent creation of a new form_id
# for further retention data
cursor.execute("""
UPDATE forms
SET author=%s,
time=%s,
state=%s
WHERE id=%s
""", (author, int(time.time()), '{}', form_id))
db.commit()
def model_vs_data_v13_log(infile,dic1, dic):
'''extract values and put them into dic (should be often updated)
'''
if not util.check_file(100, infile) : return
dic['prog_c'] = 'PHENIX'
fr=open(infile, 'r')
for ln in fr:
if ln[0:6] == 'Data: ':
for y in fr:
t=y.split(':')[1].split()
if 'Completeness in resolution range' in y:
tmp = t[0].strip()
if(float(tmp)<1.1):
dic1['comp']="%.3f" %(float(tmp)*100)
else:
dic1['comp']=tmp
elif 'Resolution range:' in y:
dic1['resh'] = t[1]
dic1['resl'] = t[0]
elif 'Number of Miller indices:' in y:
dic1['nref'] = t[0]
elif 'Wavelength' in y:
# Last one
break
elif ln[0:15] == 'Model vs Data: ':
intable = False
nnfree = 0
for y in fr:
t=y.split(':')
if "Resolution Compl Nwork" in y:
intable = True
continue
if intable:
t = y.split()
if len(t) == 0:
if nnfree:
dic1['nfree'] = nnfree
intable = False
continue
else:
nnfree += int(t[3])
continue
if 'r_work:' in y:
dic1['rfact'] = t[1].strip()
elif 'r_free:' in y:
tmp = t[1].strip()
if not util.is_number(tmp):
tmp='?'
dic1['rfree'] = tmp
elif 'Number of F-obs outliers' in y:
# End of category
break
elif ln[0:43] == 'Information extracted from PDB file header:':
for y in fr:
t=y.split(':')
if 'program_name' in y:
dic1['prog_r'] = t[1].strip()
elif 'r_work' in y:
dic1['rfact_r'] = t[1].strip()
elif 'r_free' in y:
dic1['rfree_r'] = t[1].strip()
elif 'high_resolution' in y:
dic1['resh_r'] = t[1].strip()
elif 'low_resolution' in y:
dic1['resl_r'] = t[1].strip()
elif 'exptl_method' in y:
# End of category
break
elif ln[0:44] == 'After applying resolution and sigma cutoffs:':
for y in fr:
t=y.split(':')
if 'r_work' in y:
dic1['r_cutoff'] = t[1].strip()
elif 'r_free' in y:
dic1['rf_cutoff'] = t[1].strip()
elif 'Number of F-obs outliers' in y:
# End of category
break
# print(dic['dcc'])
fr.close()
from vocabulary import Vocabulary
from util import is_number
from majorsystem import MajorSystem
if __name__ == "__main__":
major_system = MajorSystem()
vocab = Vocabulary()
prompt = "Enter a number (leave empty and press enter to exit): "
number = input(prompt)
while number != "":
if is_number(number):
pattern = major_system.build_pattern(number)
words = vocab.search(pattern)
print(f'Words for {number}: {words}\n')
else:
print("That wasn't a valid number.")
number = input(prompt)
def xtriage_log(file, dic):
''' parse data from the output of xtriage'''
if util.check_file(500, file)==0 : return
fp=open(file,'r')
dic['twin_ph'] = '?'
out1 = [0]*20
spg, twin='?', 'Y'
isigi_bin=[]
for ln in fp:
if 'Space group:' in ln[:12]:
spg=ln[12:].split('(')[0].strip()
elif 'Resolution range:' in ln[:]:
out1[0]=float(ln[17:].split()[1])
elif ' Number of centrics :' in ln[:23]:
out1[1]=int(ln[23:].split()[0])
elif ' Number of acentrics :' in ln[:23]:
out1[2]=int(ln[23:].split()[0])
elif 'p_value(height) ' in ln and ' : ' in ln :
dic['pvalue'] = util.get_value_after_id(ln,":")
elif '[MaxAnisoB-MinAnisoB]' in ln[:47]:
dic['aniso'] = util.get_value_after_id(ln,":")
elif 'max. difference between axes =' in ln:
dic['aniso'] = util.get_value_after_id(ln,"=")
elif 'The combined Z-score of ' in ln[:25]:
t=util.get_value_after_id(ln,"Z-score of")
dic['zscore'] = t.split()[0]
elif 'No ice ring related problems detected.' in ln:
dic['ice_ring'] = '0'
elif 'There were ' in ln [:20] and 'ice ring related warnings.' in ln[10:]:
dic['ice_ring'] = ln.split()[3]
elif ' Patterson function reveals a significant off-origin' in ln:
dic['stran'] = 'Y'
elif ' No significant pseudotranslation is detected' in ln:
dic['stran'] = 'N'
elif 'Acentric reflections' in ln:
n=0
for tmp in fp:
if len(tmp.strip()) <5 : continue
n=n+1
if ' <I^2>/<I>^2 :' in tmp:
out1[3] = float(tmp[19:].split()[0])
elif ' <F>^2/<F^2> :' in tmp:
out1[4] = float(tmp[19:].split()[0])
elif ' <|E^2 - 1|> :' in tmp:
out1[5] = float(tmp[19:].split()[0])
elif n>=3: break
elif 'Centric reflections' in ln:
n=0
for tmp in fp:
if len(tmp.strip()) <5 : continue
n=n+1
if ' <I^2>/<I>^2 :' in tmp:
out1[6] = float(tmp[19:].split()[0])
elif ' <F>^2/<F^2> :' in tmp:
out1[7] = float(tmp[19:].split()[0])
elif ' <|E^2 - 1|> :' in tmp:
out1[8] = float(tmp[19:].split()[0])
elif n>=3: break
elif ' Mean |L| :' in ln[:14]:
out1[9]=float(ln[14:].split()[0])
elif ' Mean L^2 :' in ln[:14]:
out1[10]=float(ln[14:].split()[0])
elif ' Multivariate Z score L-test:' in ln[:30]:
out1[11]=float(ln[30:].split()[0])
elif 'ML estimate of overall B value of ' in ln:
n=0
for tmp in fp:
n=n+1
if 'A**' in tmp: dic['bwilson']=tmp.split()[0]
if(n>=3):
dic['bwilson_s']=tmp.split()[0]
break
elif 'Statistics independent of twin laws' in ln:
n=0
for tmp in fp:
if len(tmp.strip()) <5 : continue
n=n+1
if ' <I^2>/<I>^2 :' in tmp:
dic['I2'] = tmp.split(':')[1].strip().split()[0]
elif ' <F>^2/<F^2> : ' in tmp:
dic['F2'] = tmp.split(':')[1].strip().split()[0]
elif ' <|E^2-1|> :' in tmp:
dic['E2'] = tmp.split(':')[1].strip().split()[0]
elif '<|L|>, <L^2>:' in tmp:
ttmp=tmp.split(':')[1]
dic['L'] = ttmp.split(',')[0].strip()
dic['L2'] = ttmp.split(',')[1].strip()
elif '<|L|> :' in tmp:
t= tmp.split(':')[1]
dic['L'] = t.split()[0]
elif '<L^2> :' in tmp:
t= tmp.split(':')[1]
dic['L2'] = t.split()[0]
elif 'Multivariate Z score L-test:' in tmp:
dic['Z'] = tmp.split(':')[1].strip()
elif n>=6: break
elif ('No (pseudo)merohedral twin laws were found' in ln or
(' No twinning is suspected.' in ln and
util.is_number(dic['twin_r']) and float(dic['twin_r'])>=0.3)) :
twin='N'
dic['twin_ph'] = 'N'
elif '<I/sigma_I>;_Signal_to_noise' in ln[:41]:
n=0
for tmp in fp:
n=n+1
if n>2 and '$$' in tmp[:4]: break
t=tmp.strip().split()
if len(t) ==2 : isigi_bin.append(t)
elif ' type | R obs. | Britton alpha | H alpha | ML alpha |' in ln:
n=0
for tmp in fp:
n=n+1
if n>4 : break
t=tmp.strip().split('|')
if len(t)==8 :
dic['twin_op_ph'] = t[1].strip()
dic['twin_type'] = t[2].strip()
dic['twin_r'] = t[3].strip()
dic['twin_fr_ph'] = t[6].strip()
if (dic['twin_ph'] != 'N' and util.is_number(dic['twin_r']) and
float(dic['twin_r'])<0.3) : dic['twin_ph'] = 'Y'
if len(isigi_bin)>3 :
dic['i_sigi_resh']=isigi_bin[-1][1]
if (util.is_number(isigi_bin[0][1]) and util.is_number(isigi_bin[1][1])):
a = float(isigi_bin[0][1])-float(isigi_bin[1][1])
dic['i_sigi_slop'] ='%.1f' %a
fp.close()
return spg, twin, out1
def __init__(self, n):
self.val = n
if util.is_number(n):
self.val = n.val
def get_list(file):
vlist=[]
if not util.check_file(200, file):
print('Error: file (%s) do not exist' %file)
return vlist
flist=open(file,'r').readlines()
pdbid='XXXX'
for x in flist:
if 'data_' in x:
pdbid=x.split('_')[1].strip()
break
items,values = cif.cifparse(flist, '_pdbx_density.')
sym=cif.parse_values(items,values,'_pdbx_density.space_group_name_H-M')
res=cif.parse_values(items,values,'_pdbx_density.ls_d_res_high');
rw=cif.parse_values(items,values,'_pdbx_density.R_value_R_work');
rf=cif.parse_values(items,values,'_pdbx_density.R_value_R_free');
biso=cif.parse_values(items,values,'_pdbx_density.Biso_mean');
bwil=cif.parse_values(items,values,'_pdbx_density.B_wilson');
l2=cif.parse_values(items,values,'_pdbx_density.Padilla-Yeates_L2_mean');
z=cif.parse_values(items,values,'_pdbx_density.Z_score_L_test');
fom=cif.parse_values(items,values,'_pdbx_density.fom');
isig=cif.parse_values(items,values,'_pdbx_density.I_over_sigI_resh');
isigd=cif.parse_values(items,values,'_pdbx_density.I_over_sigI_diff');
pst=cif.parse_values(items,values,'_pdbx_density.translational_pseudo_symmetry');
bsol=cif.parse_values(items,values,'_pdbx_density.B_solvent');
ksol=cif.parse_values(items,values,'_pdbx_density.K_solvent');
tlst=cif.parse_values(items,values,'_pdbx_density.partial_B_value_correction_success');
ntls=cif.parse_values(items,values,'_pdbx_density.tls_group_number');
nncs=cif.parse_values(items,values,'_pdbx_density.ncs_group_number');
nmtx=cif.parse_values(items,values,'_pdbx_density.mtrix_number');
matt=cif.parse_values(items,values,'_pdbx_density.Matthew_coeff');
solv=cif.parse_values(items,values,'_pdbx_density.solvent_content');
dpix=cif.parse_values(items,values,'_pdbx_density.Cruickshank_dpi_xyz');
rtwin=cif.parse_values(items,values,'_pdbx_density.reflns_twin');
xtwin=cif.parse_values(items,values,'_pdbx_density.twin_by_xtriage');
tmp=cif.parse_values(items,values,'_pdbx_density.iso_B_value_type');
if tmp : btype=tmp[0][0]
ctwin_t=cif.parse_values(items,values,'_pdbx_density.twin_operator');
ctwin='N'
if '2:' in ctwin_t[0] : ctwin='Y'
anis=cif.parse_values(items,values,'_pdbx_density.anisotropy');
# looped
items,values = cif.cifparse(flist, '_pdbx_density_corr.')
prog=cif.parse_values(items,values,'_pdbx_density_corr.program');
resh=cif.parse_values(items,values,'_pdbx_density_corr.ls_d_res_high');
rwork=cif.parse_values(items,values,'_pdbx_density_corr.ls_R_factor_R_work');
rfree=cif.parse_values(items,values,'_pdbx_density_corr.ls_R_factor_R_free');
fcc=cif.parse_values(items,values,'_pdbx_density_corr.correlation_coeff_Fo_to_Fc');
rsr=cif.parse_values(items,values,'_pdbx_density_corr.real_space_R');
dcc=cif.parse_values(items,values,'_pdbx_density_corr.correlation');
detail=cif.parse_values(items,values,'_pdbx_density_corr.details');
nr, nc=0, 0
for i, x in enumerate(detail):
if 'Best' in x :
nc=i
break
rprog, cprog = prog[nr].replace(' ', ''), prog[nc]
crw, crf, fcc, rsr, dcc=rwork[nc], rfree[nc], fcc[nc],rsr[nc],dcc[nc]
rw_crw='?'
if util.is_number(rw[0]) and util.is_number(crw):
t=int (1000*(float(rw[0]) -float(crw)))
rw_crw='%d' %(t)
all=[pdbid, res,rw_crw, rw, rf, crw, crf, fcc, rsr, dcc, fom, biso,bwil,
matt,solv, ksol, bsol, ntls,nncs, nmtx,
tlst,btype, pst, rtwin,xtwin, ctwin, l2, z,anis, isig,isigd, rprog, sym]
all_new=[]
for x in all:
t=x
if not x :
t='?'
else:
if (type(x)==list ):t=x[0]
y=t.replace(' ', '_')
if util.is_number(y) and '.' in y:
y='%.2f' %float(y)
all_new.append(y)
return all_new
def removable_words(words):
return [
i for i, w in enumerate(words)
if len(w) > 0 and not (is_punctuation(w) or is_number(w))
]
def write_table(self, path):
if not self.page_tables: return
# page sort
sort_page = [p for p in self.page_tables.keys()]
sort_page.sort()
# set cell covers
i = 1
for page in sort_page:
tables = self.page_tables[page]
self.page_tables[page] = []
try:
for t in tables:
info = self.get_table_info(t, page)
tt = table_structure.Table(t, self.html)
tt.info = info
tt.set_table_cells(i)
# pdb.set_trace()
i += tt.size[0] + 2
self.page_tables[page].append(tt)
except:
print page
print tables
# write tables in excel
wb = xlwt.Workbook()
sheet = wb.add_sheet('tables', cell_overwrite_ok=True)
j = 0
for page in sort_page:
tables = self.page_tables[page]
try:
for table in tables:
sheet.write(j, 0, "page " + str(page + 1))
sheet.write(j, 1, table.info)
j += table.size[0] + 2
for cell in table.cells:
if cell.covers:
if len(cell.covers) > 1:
top_row = cell.covers[0][0]
bottom_row = cell.covers[-1][0]
left_column = cell.covers[0][1]
right_column = cell.covers[-1][1]
if util.is_number(
cell.text
): # if cell.text is number, save in number style
content = cell.text.replace(',', '')
if content.isdigit():
sheet.write_merge(
top_row, bottom_row, left_column,
right_column, float(content))
else:
sheet.write_merge(
top_row, bottom_row, left_column,
right_column, cell.text)
else:
sheet.write_merge(top_row, bottom_row,
left_column,
right_column, cell.text)
else:
if util.is_number(cell.text):
content = cell.text.replace(',', '')
if content.isdigit():
sheet.write(cell.covers[0][0],
cell.covers[0][1],
float(content))
else:
sheet.write(cell.covers[0][0],
cell.covers[0][1],
cell.text)
else:
sheet.write(cell.covers[0][0],
cell.covers[0][1], cell.text)
except:
print page
wb.save(path)
