def getDailyWeatherDataForPrevision(tempMean, t): db = sqlite3.connect(db_path_string) cursor = db.cursor() t_day = int(t - (t % (24*3600))) - 3600 # weather uses UTC + 2 time t_next_day = t_day + 3600*24 #print 'timestamp for daily weather', t_day data = cursor.execute('SELECT avg(temperatur), sum(precipitation) FROM weatherForecast WHERE timestamp>=:t_min and timestamp<=:t_max',\ {"t_min":t_day, "t_max":t_next_day}).fetchone() if not util.is_number(data[0]): #print 'no temperature data for this day, assuming average temperature' dayTemperatureAvg = tempMean else: dayTemperatureAvg = float(data[0]) if not util.is_number(data[1]): #print 'no precipitation data for this day, assuming no precipitations' dayPrecipitationTotal = 0.0 else: dayPrecipitationTotal = float(data[1]) db.close() return [dayTemperatureAvg, dayPrecipitationTotal]
def eq(self, a, b): ''' operator "=" If "=" is applied to two equal blocks or two equal integers, then the result equals 1, otherwise 0''' if not (util.is_number(a) and util.is_number(b)) and not(self.is_block(a) and self.is_block(b)): raise ValueError("Operator '=' can only be applied to two blocks or two integers!") return a == b
def write_table(self,path): if not self.page_tables: return # page sort sort_page = [p for p in self.page_tables.keys()] sort_page.sort() # set cell covers i = 1 for page in sort_page: tables = self.page_tables[page] self.page_tables[page] = [] for t in tables: info = self.get_table_info(t,page) tt = table_structure.Table(t) tt.info = info tt.set_table_cells(i) # pdb.set_trace() i += tt.size[0]+2 self.page_tables[page].append(tt) # write tables in excel wb = xlwt.Workbook() sheet = wb.add_sheet('tables',cell_overwrite_ok=True) j = 0 for page in sort_page: tables = self.page_tables[page] for table in tables: sheet.write(j,0,"page "+str(page+1)) sheet.write(j,1,table.info) j += table.size[0]+2 for cell in table.cells: if cell.covers: if len(cell.covers)>1: top_row = cell.covers[0][0] bottom_row = cell.covers[-1][0] left_column = cell.covers[0][1] right_column = cell.covers[-1][1] if util.is_number(cell.text): # if cell.text is number, save in number style content = cell.text.replace(',','') if content.isdigit(): sheet.write_merge(top_row, bottom_row, left_column, right_column, float(content)) else: sheet.write_merge(top_row, bottom_row, left_column, right_column, cell.text) else: sheet.write_merge(top_row, bottom_row, left_column, right_column, cell.text) else: if util.is_number(cell.text): content = cell.text.replace(',','') if content.isdigit(): sheet.write(cell.covers[0][0],cell.covers[0][1],float(content)) else: sheet.write(cell.covers[0][0],cell.covers[0][1],cell.text) else: sheet.write(cell.covers[0][0],cell.covers[0][1],cell.text) wb.save(path)
def group(self): ''' operation "g" ''' arg1 = self.data.pop() # the argument on top of the stack will be at the end of the block arg2 = self.data.pop() if self.is_block(arg1) and self.is_block(arg2): # two blocks block = arg2[0:len(arg2) - 1] arg1 = arg1[1:len(arg1)] if util.is_number(arg2[-2]) and util.is_number(arg1[0]): block += " " block += arg1 elif util.is_number(arg1) and util.is_number(arg2): # two integers block = "[" + str(arg2) + " " + str(arg1) + "]" elif util.is_number(arg2) and self.is_block(arg1): # interger (arg2) and block (arg1) block = "[" + str(arg2) arg1 = arg1[1:len(arg1)] if util.is_number(arg1[0]): block += " " block += arg1 elif self.is_block(arg2) and util.is_number(arg1): # block (arg2) and integer (arg1) block = arg2[0:len(arg2) - 1] if util.is_number(arg2[-2]): block += " " block += str(arg1) + "]" else: raise ValueError("todo") #TODO: remove debug # print "group block: '{0}'".format(block) self.data.push(block)
def getVacationData(): #period of the holiday data :2014-2015 start = int(util.datetimeToTimestamp(datetime.datetime(2014,01,01))) end = int(util.datetimeToTimestamp(datetime.datetime(2015,12,31))) step = 3600 * 24 vacation = dict() with vacation_data_path.open('r') as f: read_data = f.read() vacationDayIds = read_data.split(';') # add all vacation days for dayId in vacationDayIds: if util.is_number(dayId): dayId = timestampToDayId(float(dayId)) vacation[dayId] = 1 # add remaining days for dayId2 in range(start , end, step): dayId2 = timestampToDayId(float(dayId2)) if not dayId2 in vacation: vacation[dayId2] = 0 return vacation
def apply_crypt(string_const): """Apply the crypto routine to the string costant""" if string_const is None: return '' if not u.is_number(string_const): return 'a7ign' + u.crypt_string(string_const) else: return string_const
def getDailyWeatherData(): db = sqlite3.connect(db_path_string) cursor = db.cursor() weather_data = cursor.execute('SELECT * FROM weather_day').fetchall() temperatures, precipitations = list(), list() #temperatureTotal, temperatureCount, precipitationTotal, precipitationCount = 0 , 0 , 0 , 0 for data in weather_data: if util.is_number(data[1]) : #temperatureTotal = temperatureTotal + float(data[1]) #temperatureCount = temperatureCount + 1 temperatures.append(float(data[1])) if util.is_number(data[2]) : #precipitationTotal = precipitationTotal + float(data[2]) #precipitationCount = precipitationCount + 1 precipitations.append(float(data[2])) tempMean = numpy.mean(temperatures) tempStdDev = numpy.std(temperatures) precStdDev = numpy.std(precipitations) #print tempMean, tempStdDev, precStdDev normalizedTemperatures, normalizedPrecipitations = dict(), dict() for data2 in weather_data: if util.is_number(data2[1]) : #temperatureTotal = temperatureTotal + float(data2[1]) #temperatureCount = temperatureCount + 1 dayId = timestampToDayId(data2[0]) normalizedTemperatures[dayId] = (float(data2[1]) - tempMean ) / tempStdDev if util.is_number(data2[2]) : #precipitationTotal = precipitationTotal + float(data2[2]) #precipitationCount = precipitationCount + 1 dayId = timestampToDayId(data2[0]) normalizedPrecipitations[dayId] = float(data2[2]) / precStdDev db.close() return [tempMean, tempStdDev, precStdDev, normalizedTemperatures, normalizedPrecipitations]
def read(self): # read from input stream # item = chr(self.iS.read()) item = self.iS.read() # only digits, commands and operators may be read if util.is_number(item): # self.data.push(int(item)) self.data.push(item) elif item in self.operations: raise ValueError("Input {0} is not a valid char code")
def write(self): '''operation "w"''' # make sure that the top element of the data stack is an integer or a command val = self.data.pop() # print ">> {0}".format(val) if not util.is_number(val) and val not in self.operations: raise ValueError("Cannot write to output stream" + val) # for byte in to_ascii(val): # self.oS.write(byte) self.oS.write(val)
def get_tracform_meta(self, src, db=None): """ Returns the meta information about a form based on a form id (int or long) or context (parent realm, parent id, TracForms subcontext). """ db = self._get_db(db) cursor = db.cursor() sql = """ SELECT id, realm, resource_id, subcontext, author, time, keep_history, track_fields FROM forms """ if not is_number(src): sql += """ WHERE realm=%s AND resource_id=%s AND subcontext=%s """ else: sql += """ WHERE id=%s """ form_id = None realm = None resource_id = None subcontext = None if not is_number(src): realm, resource_id, subcontext = src else: form_id = src src = tuple([src],) cursor.execute(sql, src) return cursor.fetchone() or \ (form_id, realm, resource_id, subcontext, None, None, None, None)
def generate_features(name): data = pandas.read_csv("homesite/{}.csv".format(name)) processed_data = pandas.DataFrame() onehot_encode_fields = [] dropped_fields = ["PersonalField16", "PersonalField17", "PersonalField18", "PersonalField19"] for k in data.keys(): if is_number(data[k][1]): processed_data[k] = data[k] elif k == "Original_Quote_Date": months = [] years = [] for d in data[k]: dt = datetime.datetime.strptime(d, "%Y-%m-%d") months.append(dt.month) years.append(dt.year) processed_data["Month"] = pandas.Series(months) processed_data["Year"] = pandas.Series(years) else: if k in dropped_fields: continue onehot_encode_fields.append(k) df = encode_onehot(data, cols=onehot_encode_fields) for k in df.keys(): if is_number(df[k][1]): print k processed_data[k] = df[k].apply(lambda x: locale.atof(x) if isinstance(x, basestring) else x) elif k == "Original_Quote_Date": months = [] years = [] for d in df[k]: dt = datetime.datetime.strptime(d, "%Y-%m-%d") months.append(dt.month) years.append(dt.year) processed_data["Month"] = pandas.Series(months) processed_data["Year"] = pandas.Series(years) processed_data.to_csv("homesite/processed_features{}.csv".format(name), quotechar='"')
def get_tracform_fields(self, src, db=None): db = self._get_db(db) cursor = db.cursor() if is_number(src): form_id = src else: form_id = self.get_tracform_meta(src, db=db)[0] cursor.execute(""" SELECT field, author, time FROM forms_fields WHERE id=%s """, (form_id,)) return cursor.fetchall()
def get_tracform_fields(self, src, db=None): db = self._get_db(db) cursor = db.cursor() if is_number(src): form_id = src else: form_id = self.get_tracform_meta(src, db=db)[0] cursor.execute( """ SELECT field, author, time FROM forms_fields WHERE id=%s """, (form_id, )) return cursor.fetchall()
def getNearestWeatherPastForRegression(F, hourLimit, db_path): db = sqlite3.connect(db_path) cursor = db.cursor() R, toRemove = dict(), list() for time in F: min_time = time-3600*hourLimit data = cursor.execute('''SELECT timestamp, precipitation FROM Weather WHERE timestamp>:lowLimit AND timestamp<=:highLimit ORDER BY timestamp DESC''',{"lowLimit":min_time, "highLimit":time}).fetchone() if data is None: toRemove.append(int(time)) elif not util.is_number(data[1]): toRemove.append(int(time)) else: R[int(time)] = float(data[1]) return [R, toRemove]
def get_tracform_history(self, src, db=None): db = self._get_db(db) cursor = db.cursor() if is_number(src): form_id = src else: form_id = self.get_tracform_meta(src, db=db)[0] cursor.execute(""" SELECT author, time, old_state FROM forms_history WHERE id=%s ORDER BY time DESC """, (form_id,)) return cursor.fetchall()
def create_simplified_projections(): for s in ['fanduel', 'draftkings']: conf.site = s conf.rotogrinder_hitter_path =\ 'http://rotogrinders.com/projected-stats/mlb-hitter.csv?site=%s' % conf.site conf.rotogrinder_pitcher_path =\ 'http://rotogrinders.com/projected-stats/mlb-pitcher.csv?site=%s' % conf.site sals_and_pos = retrieve_mlb_player_salaries_and_positions() if conf.batting_orders is None: conf.batting_orders, conf.pitchers = retrieve_mlb_batting_order() df = pd.read_csv('./projections/%s_%s_%s.csv' % (conf.site, conf.proj_date, conf.proj_iteration), index_col=0) idx = df.index[:] means = [] for p in idx: means.append([np.mean(df.loc[p][0:conf.simulated_game_count])]) mean_df = pd.DataFrame(means, index=idx, columns=['mean']) df = df.join(mean_df) # join our two data frames projections_overall = df.join(sals_and_pos, how='left') # drop players that weren't included to_drop = [] for name in projections_overall.index: if type(projections_overall.at[name, 'cost']) == np.ndarray: print name print projections_overall.loc[name] pdb.set_trace() if not is_number(projections_overall.at[name, 'cost']): to_drop.append(name) elif name in conf.excluded_batters or name in\ conf.excluded_pitchers: to_drop.append(name) elif conf.use_inclusion is False and projections_overall.at[ name, 'team'] in conf.excluded_teams: to_drop.append(name) elif conf.use_inclusion is True and projections_overall.at[ name, 'team'] not in conf.included_teams: to_drop.append(name) projections_overall = projections_overall.drop(to_drop) projections_overall['exclude'] = 0 projections_overall[['pos', 'mean', 'cost', 'team', 'exclude']].to_csv( './simplified_projections/%s_%s_%s.csv' % (conf.site, conf.proj_date, conf.proj_iteration))
def get_tracform_history(self, src, db=None): db = self._get_db(db) cursor = db.cursor() if is_number(src): form_id = src else: form_id = self.get_tracform_meta(src, db=db)[0] cursor.execute( """ SELECT author, time, old_state FROM forms_history WHERE id=%s ORDER BY time DESC """, (form_id, )) return cursor.fetchall()
def push_code(self, s): '''push code onto the calculator's code stack. Left-most command will be at the top of the code stacks''' i = 0 l = [] while i < len(s): if util.is_number(s[i]): n = int(s[i]) while i + 1 < len(s) and util.is_number(s[i + 1]): i = i + 1 n = n * 10 + int(s[i]) l.append(n) elif s[i] == "[": end = i cnt = 1 while cnt != 0 or s[end] != "]": end += 1 if s[end] == "[": cnt += 1 elif s[end] == "]": cnt -= 1 block = s[i:end + 1] i = end l.append(block) elif s[i] in self.operations: l.append(s[i]) elif s[i].isspace(): pass else: msg = "Invalid code : " + s[i:] raise ValueError(msg) i = i + 1 for i in reversed(l): self.code.push(i)
def get_tracform_fieldinfo(self, src, field, db=None): """Retrieve author and time of last change per field.""" db = self._get_db(db) cursor = db.cursor() if is_number(src): form_id = src else: form_id = self.get_tracform_meta(src, db=db)[0] cursor.execute(""" SELECT author, time FROM forms_fields WHERE id=%s AND field=%s """, (form_id, field)) return cursor.fetchone() or (None, None)
def get_tracform_fieldinfo(self, src, field, db=None): """Retrieve author and time of last change per field.""" db = self._get_db(db) cursor = db.cursor() if is_number(src): form_id = src else: form_id = self.get_tracform_meta(src, db=db)[0] cursor.execute( """ SELECT author, time FROM forms_fields WHERE id=%s AND field=%s """, (form_id, field)) return cursor.fetchone() or (None, None)
def voting_matrix(): voting_matrix = [] for row in rollcall: # just to stay consistent, we format the party as a list mps.append((row[:36], [row[20:23]])) votes = row[36:] votes = [convert_sen(thing) for thing in votes if is_number(thing)] voting_matrix.append(votes) open(sen_filename_voting_mat, "w").close() mat_f = open(sen_filename_voting_mat, "wb") pickle.dump(voting_matrix, mat_f) mat_f.close() open(sen_filename_mps, "w").close() mps_f = open(sen_filename_mps, "wb") pickle.dump(mps, mps_f) mps_f.close()
def process_line(self, l_info, unnamed=True): if isinstance(l_info, str) and not is_number(l_info): assert (Line(l_info) in self.lines) return Line(l_info) if not isinstance(l_info, tuple): raise NotImplementedError( f"[process_line] l_info must be tuple or string") l_pred = l_info[0].lower() l_args = l_info[1:] l_val = None if l_pred == "line": assert (len(l_args) == 2) ps = [self.process_point(p) for p in l_args] l_val = FuncInfo("connecting", ps) elif l_pred in ["perp-at", "para-at"]: assert (len(l_args) == 2) p = self.process_point(l_args[0]) l = self.process_line(l_args[1]) l_val = FuncInfo(l_pred, [p, l]) elif l_pred == "perp-bis": assert (len(l_args) == 2) ps = [self.process_point(p) for p in l_args] l_val = FuncInfo(l_pred, ps) elif l_pred in ["isogonal", "isotomic"]: assert (len(l_args) == 4) ps = [self.process_point(p) for p in l_args] l_val = FuncInfo(l_pred, ps) elif l_pred in ["i-bisector", "e-bisector"]: assert (len(l_args) == 3) ps = [self.process_point(p) for p in l_args] l_val = FuncInfo(l_pred, ps) elif l_pred == "reflect-ll": assert (len(l_args) == 2) ls = [self.process_line(l) for l in l_args] l_val = FuncInfo(l_pred, ls) if l_val is not None: L = Line(l_val) if unnamed: self.unnamed_lines.append(L) return L else: raise NotImplementedError( f"[process_line] Unsupported line pred: {l_pred}")
def _get_mdf_filename(self): if self.mosmask is not None: mdffile = self.mosmask # Expand the MOS mask number if is_number(self.mosmask): observatory, semester, prgtype, queuenum = self.get_program_id_parts() mdffile = "%s%s%s%03i-%02i" % (observatory, semester, prgtype, queuenum, int(self.mosmask)) debug("...mosmask =", mdffile) else: mdffile = self.focal_plane_mask() mdffile = fits_filename(mdffile) #----------------------------------------------------------------------- # Start searching around willy nilly for the MDF file if os.path.exists(mdffile): return mdffile # note, the order in which directories are added to this list gives priority dirs = [] if self.mdfdir is not None: dirs.append(self.mdfdir) dname = os.path.dirname(self.filename) if dname and dname != ".": dirs.append(dname) dirs.append(os.getcwd()) # search through semester directories as well semester_dir = self.get_observatory_prefix() + self.get_semester() directories_to_search = [] for dname in dirs: directories_to_search.append(dname) dname = os.path.join(dname, semester_dir) directories_to_search.append(dname) # now search through the directories for dname in directories_to_search: fname = os.path.join(dname, mdffile) debug("...trying", fname) if os.path.exists(fname): return fname raise ValueError("Unable to find MDF file named '%s'" % mdffile)
def get_relates_by_spu_draft_id(spu_draft_id): result = dict() sku_draft_ids = get_db_sku_ids(spu_draft_id) result["spuDraftId"] = spu_draft_id result["skuDraftIds"] = sku_draft_ids spu_id = get_db_spu_ids(spu_draft_id, False) if not (spu_id is None) and bm.is_number(spu_id): sku_ids = get_db_sku_ids(spu_id, False) sku_data = mu.get_db_group_info("db_pop.item_sku", "id", sku_ids) item_ids = list(map(lambda x: x['item_id'], sku_data)) item_ids = list(set(item_ids)) result["spuId"] = spu_id result["skuIds"] = sku_ids result["itemIds"] = item_ids return result
def __init__(self, civicrm, **kwargs): self.civicrm = civicrm if 'contact' in kwargs: contact = kwargs['contact'] elif 'member_id' in kwargs: contact = civicrm.get('Contact', external_identifier=kwargs['member_id'])[0] else: raise ValueError('now loading argument stated') self.civi_id = int(contact['id']) if is_number(contact['external_identifier']): self.member_id = int(contact['external_identifier']) else: self.member_id = 0 self.firstname = contact['first_name'] self.lastname = contact['last_name'] self.email = contact['email'] self.country = contact['country'] self.city = contact['city'] self.street = contact['street_address'] self.postalcode = contact['postal_code'] self.phone = contact['phone'] self.state = contact['state_province_name'] if 'memberships' in kwargs: if kwargs['memberships'] != None: self.memberships = list(); for membership in kwargs['memberships']: if membership.contact_id == self.civi_id: self.memberships.append(membership) else: self.memberships = list(); membership_values = civicrm.get('Membership', contact_id=self.civi_id) for membership_data in membership_values: membership = Membership(data=membership_data) self.memberships.append(membership) if 'verification' in kwargs and kwargs['verification'] == True: self.verified = False verification_values = civicrm.get('CustomValue', entity_id=self.civi_id) for verification_data in verification_values: if verification_data['id'] == '7' and verification_data['0'] == '1': self.verified = True
def process_number(self, n_info): if isinstance(n_info, str) and n_info.lower() == "pi": return Num(math.pi) if isinstance(n_info, str) and is_number(n_info): return Num(float(n_info)) if not isinstance(n_info, tuple): raise NotImplementedError( f"[process_number] n_info must be tuple or string") n_pred = n_info[0].lower() n_args = n_info[1:] if n_pred == "dist": assert (len(n_args) == 2) p1 = self.process_point(n_args[0]) p2 = self.process_point(n_args[1]) n_val = FuncInfo("dist", [p1, p2]) return Num(n_val) elif n_pred in ["uangle", "area"]: assert (len(n_args) == 3) p1, p2, p3 = [self.process_point(p) for p in n_args] n_val = FuncInfo(n_pred, [p1, p2, p3]) return Num(n_val) elif n_pred == "radius": assert (len(n_args) == 1) circ = self.process_circle(n_args[0]) n_val = FuncInfo("radius", [circ]) return Num(n_val) elif n_pred == "diam": assert (len(n_args) == 1) circ = self.process_circle(n_args[0]) n_val = FuncInfo("diam", [circ]) return Num(n_val) elif n_pred in ["div", "add", "sub", "mul", "pow"]: assert (len(n_args) == 2) n1, n2 = [self.process_number(n) for n in n_args] n_val = FuncInfo(n_pred, [n1, n2]) return Num(n_val) elif n_pred in ["neg", "sqrt"]: assert (len(n_args) == 1) n = self.process_number(n_args[0]) n_val = FuncInfo(n_pred, [n]) return Num(n_val) else: raise NotImplementedError( f"[process_number] Unsupporrted number pred: {n_pred}")
def insert_course(db, headings, info): """Add a row to the Course table. Args: headings (List[str]): headings for the current CSV file. info (List[str]): [city, category, season]. Returns: int: The ID of the current course. """ city = info[0] if info[0] != 'Stage' else 'Las Vegas' cat = info[1] if not is_number(info[1]) else 'Stage ' + info[1] course_id = db.query_file('data/sql/insert_course.sql', city=city, cat=cat, s=info[2]).all() return course_id[0].course_id
def availableCyclicMean(type, stationId, thresholdMinutes): available, numberOfWeeks, availableCyclicMean = list(), list(), list() db = sqlite3.connect(db_path_string) cursor = db.cursor() if not type in ['bike', 'stand']: print type return availableCyclicMean if type == 'bike': stationAvailabilityData = cursor.execute( 'SELECT timestamp, availableBikes FROM OldResults WHERE stationId =:stationId', {"stationId": stationId}) if type == 'stand': stationAvailabilityData = cursor.execute( 'SELECT timestamp, availableStand FROM OldResults WHERE stationId =:stationId', {"stationId": stationId}) indices = weekCycleIndices(thresholdMinutes) for indice in indices: numberOfWeeks.insert(indice, 0) available.insert(indice, 0) for data in stationAvailabilityData: indice = timestampToWeekCycleIndice(data[0], thresholdMinutes) if not util.is_number(data[1]): print 'unknown availability data', data[1] available[indice] = available[indice] + data[1] numberOfWeeks[indice] += 1 for indice in indices: if numberOfWeeks[indice] == 0: availableCyclicMean.insert(indice, 0) else: availableCyclicMean.insert( indice, round(float(available[indice]) / numberOfWeeks[indice], 2)) db.close() return availableCyclicMean
def auto_format(data): '''data is a single list [ ,,]; return string ''' ss = [] for x in data: if util.is_number(x): if math.fabs(x) < 1: ss.append(' %.3f ' % x) elif 1.0 <= math.fabs(x) < 10: ss.append(' %.2f ' % x) elif 10 <= math.fabs(x) < 100.: ss.append(' %.1f ' % x) elif math.fabs(x) >= 100.: ss.append(' %.0f ' % (x + 0.0001)) else: ss.append(' %s ' % x) return ' '.join(ss)
def create_simplified_projections(): for s in ['fanduel', 'draftkings']: conf.site = s conf.rotogrinder_hitter_path =\ 'http://rotogrinders.com/projected-stats/mlb-hitter.csv?site=%s' % conf.site conf.rotogrinder_pitcher_path =\ 'http://rotogrinders.com/projected-stats/mlb-pitcher.csv?site=%s' % conf.site sals_and_pos = retrieve_mlb_player_salaries_and_positions() if conf.batting_orders is None: conf.batting_orders, conf.pitchers = retrieve_mlb_batting_order() df = pd.read_csv('./projections/%s_%s_%s.csv' % (conf.site, conf.proj_date, conf.proj_iteration), index_col=0) idx = df.index[:] means = [] for p in idx: means.append([np.mean(df.loc[p][0:conf.simulated_game_count])]) mean_df = pd.DataFrame(means, index=idx, columns=['mean']) df = df.join(mean_df) # join our two data frames projections_overall = df.join(sals_and_pos, how='left') # drop players that weren't included to_drop = [] for name in projections_overall.index: if type(projections_overall.at[name, 'cost']) == np.ndarray: print name print projections_overall.loc[name] pdb.set_trace() if not is_number(projections_overall.at[name, 'cost']): to_drop.append(name) elif name in conf.excluded_batters or name in\ conf.excluded_pitchers: to_drop.append(name) elif conf.use_inclusion is False and projections_overall.at[name, 'team'] in conf.excluded_teams: to_drop.append(name) elif conf.use_inclusion is True and projections_overall.at[name, 'team'] not in conf.included_teams: to_drop.append(name) projections_overall = projections_overall.drop(to_drop) projections_overall['exclude'] = 0 projections_overall[['pos', 'mean', 'cost', 'team', 'exclude']].to_csv('./simplified_projections/%s_%s_%s.csv' % (conf.site, conf.proj_date, conf.proj_iteration))
def get_pop_corr(infile, nn, narg, arg): '''nn, the nth argument; narg, the maximum argument; arg, the argument ''' xxrange, yyrange, title, xlabel, ylabel, xcol, ycol, nstep = '', '', '', '', '', 0, 0, 0 ss = '' for i in range(nn, narg): ss = ss + '%s ' % arg[i] ss_split = ss.split(',') for x in ss_split: s = x.strip().split('=') key = s[0].strip().lower() if ('xcol' == key or 'ycol' == key) and not util.is_number(s[1]): print 'Error: not an integer after xcol or ycol. both must be separated by ",".' return if 'xrange' == key: xxrange = s[1] elif 'yrange' == key: yyrange = s[1] elif 'title' == key: title = s[1] elif 'xlabel' == key: xlabel = s[1] elif 'ylabel' == key: ylabel = s[1] elif 'xcol' == key: xcol = int(s[1]) elif 'ycol' == key: ycol = int(s[1]) elif 'nstep' == key: nstep = int(s[1]) print 'Input file=%s' % infile if xcol > 0 and ycol > 0: #do correlations correlation_plot(infile, xcol, ycol, xlabel, ylabel, title, nstep, xxrange, yyrange) elif xcol > 0: histogram_plot(infile, xcol, xlabel, ylabel, title, nstep, xxrange)
def getNearestPrecipitationsForPrevision(times, hourLimit, db_path): db = sqlite3.connect(db_path) cursor = db.cursor() R = dict() for t in times: min_time = t-3600*hourLimit data = cursor.execute('''SELECT timestamp, precipitation FROM WeatherForecast WHERE timestamp>:lowLimit AND timestamp<=:highLimit ORDER BY timestamp DESC''',{"lowLimit":min_time, "highLimit":t}).fetchone() if data and util.is_number(data[1]): R[t] = float(data[1]) #print 'found precipitation data', data[1] #else: #print 'precipitation data not found' db.close() return R
def execute(self): '''execute the "program" on the code stack''' while (not self.code.isEmpty()): #TODO: remove debug # print "----]] stack: {0} ^^^ {1} ".format(self.data.toString(True), self.code.toString()) # print "----]] stack: {0} ^^^ {1} ".format(self.data.toString(True,80), self.code.toString(False,50)) token = self.code.pop() if util.is_number(token): # integers are simply pushed onto the data stack self.data.push(int(token)) elif token in self.binaryOps: # binary operators: {+,-,*,/,%,&,|,=,<,>} # pop to items from data stack a = self.data.pop() b = self.data.pop() #TODO: remove debug # print "bin op={0}, a={1}, b={2}".format(token,a,b) # get the operator op_func = self.binaryOps[token] if(token != "="): # "=" does not require a and b to be integers #check that they are integers util.expectInt(a) util.expectInt(b) # apply the operator... c = int(op_func(a, b)) # some operators (i.e., "<" and ">" return "True" or "False"; so parse it to int # and push the result onto the data stack self.data.push(c) elif token == "~": # negation val = int(self.data.pop()) self.data.push(-val) elif token == "x": return #quick fix: exit gracefully elif token in self.commands: func = self.commands[token] func() elif token.startswith("[") and token.endswith("]"): self.data.push(token) else: msg = "Unknown command: " + token raise ValueError(msg)
def process_circle(self, c_info, unnamed=True): if isinstance(c_info, str) and not is_number(c_info): assert (Circle(c_info) in self.circles) return Circle(c_info) if not isinstance(c_info, tuple): raise NotImplementedError( f"[process_circle] c_info must be tuple or string") c_pred = c_info[0].lower() ps = [self.process_point(p) for p in c_info[1:]] c_val = None if c_pred == "circ": assert (len(ps) == 3) c_val = FuncInfo("c3", ps) elif c_pred == "coa": assert (len(ps) == 2) c_val = FuncInfo(c_pred, ps) elif c_pred == "diam": assert (len(ps) == 2) c_val = FuncInfo("diam", ps) elif c_pred == "circumcircle": assert (len(ps) == 3) c_val = FuncInfo("circumcircle", ps) elif c_pred == "incircle": assert (len(ps) == 3) c_val = FuncInfo("incircle", ps) elif c_pred == "excircle": assert (len(ps) == 3) c_val = FuncInfo("excircle", ps) elif c_pred == "mixtilinear-incircle": assert (len(ps) == 3) c_val = FuncInfo(c_pred, ps) if c_val is not None: C = Circle(c_val) if unnamed: self.unnamed_circles.append(C) return C else: raise NotImplementedError( f"[process_circle] Unsupported circle pred: {c_pred}")
def get_tracform_state(self, src, db=None): db = self._get_db(db) cursor = db.cursor() sql = """ SELECT state FROM forms """ if not is_number(src): sql += """ WHERE realm=%s AND resource_id=%s AND subcontext=%s """ else: sql += """ WHERE id=%s """ src = tuple([src],) cursor.execute(sql, src) row = cursor.fetchone() return row and row[0]
def get_tracform_state(self, src, db=None): db = self._get_db(db) cursor = db.cursor() sql = """ SELECT state FROM forms """ if not is_number(src): sql += """ WHERE realm=%s AND resource_id=%s AND subcontext=%s """ else: sql += """ WHERE id=%s """ src = tuple([src], ) cursor.execute(sql, src) row = cursor.fetchone() return row and row[0]
def describe_instances(self, instance_ids): print "========= A potential variation for probing the region ==========" d = defaultdict(list) for instance_id in instance_ids: assert (instance_id < len(self.rows)) row = self.rows[instance_id] for feature in self.feature_names: value = row[feature] if is_number(value): value = float(value) if float(value) == -1.0: continue feature = remove_XXX(feature) d[feature].append(value) for feature in self.feature_names: feature = remove_XXX(feature) values = d[feature] if values and type(values[0]) is float: lo, hi = min(values), max(values) if lo == hi: continue # no variation elif lo == 0 and hi == 1: print " * {0:<13}: [{1:}, {2:}]".format( feature, "no", "yes") else: unit = get_unit(feature) print " * {0:<13}: [{1:}, {2:}] {3:}".format( feature, lo, hi, unit) elif values and type(values[0]) is str: values = list(set(values)) for i in range(0, len(values), 3): vals = ", ".join(values[i:i + 3]) if i == 0: print " * {0:<13}: {1:}".format(feature, vals) else: print " * {0:<13} {1:}".format("", vals)
def availableCyclicMean(type, stationId, thresholdMinutes): available, numberOfWeeks, availableCyclicMean = list(), list(), list() db = sqlite3.connect(db_path_string) cursor = db.cursor() if not type in ['bike','stand'] : print type return availableCyclicMean if type == 'bike': stationAvailabilityData = cursor.execute('SELECT timestamp, availableBikes FROM OldResults WHERE stationId =:stationId', { "stationId": stationId}) if type == 'stand': stationAvailabilityData = cursor.execute('SELECT timestamp, availableStand FROM OldResults WHERE stationId =:stationId', { "stationId": stationId}) indices = weekCycleIndices(thresholdMinutes) for indice in indices: numberOfWeeks.insert(indice, 0) available.insert(indice, 0) for data in stationAvailabilityData: indice = timestampToWeekCycleIndice(data[0], thresholdMinutes) if not util.is_number(data[1]): print 'unknown availability data', data[1] available[indice] = available[indice] + data[1] numberOfWeeks[indice] += 1 for indice in indices: if numberOfWeeks[indice] == 0: availableCyclicMean.insert(indice,0) else: availableCyclicMean.insert(indice, round(float(available[indice])/numberOfWeeks[indice],2)) db.close() return availableCyclicMean
def find_xyzlim_compound(compid, coord): '''find xyzlimit used by mapmask, and write the coord in cif or pdb format. compid: atom_group_id (model_compound_chainID_resnumber_alter_insertion) coord: the coordinate file idd = 0, cif format; =1, the pdb format ''' comp = 'XXXX' t1 = compid.split(':') for i, x in enumerate(t1): t = x.split('_') if i == 0: comp = '_'.join([t[0], t[1], t[2], t[3]]) if len(t) != 6: print( 'Error: in group-id (%d). it should be (model_compound_chainID_resnumber_alter_insertion).' % (i + 1)) return '', '' idd = util.is_cif(coord) xyzcomp = comp + '.pdb' if idd == 1: xyzcomp = comp + '.cif' fw = open(xyzcomp, 'w') border = 1 #extend a little to cover more density xx, yy, zz = [], [], [] if idd == 1: #input cif format fw.write('data_xyzcomp\n#\n') flist = open(coord, 'r').readlines() items, values = cif.cifparse(flist, '_cell.') fw.write('\n#\n') for m, p in enumerate(items): fw.write("%s %s\n" % (p, values[m])) cell = cif.get_cell(flist) items, values = cif.cifparse(flist, '_atom_site.') comp = cif.parse_values(items, values, "_atom_site.auth_comp_id") asym = cif.parse_values(items, values, "_atom_site.auth_asym_id") seq = cif.parse_values(items, values, "_atom_site.auth_seq_id") alt = cif.parse_values(items, values, "_atom_site.label_alt_id") ins = cif.parse_values(items, values, "_atom_site.pdbx_PDB_ins_code") x = cif.parse_values(items, values, "_atom_site.Cartn_x") y = cif.parse_values(items, values, "_atom_site.Cartn_y") z = cif.parse_values(items, values, "_atom_site.Cartn_z") model = cif.parse_values(items, values, "_atom_site.pdbx_PDB_model_num") if (not (alt and comp and ins and asym and seq and x and y and z)): print( 'Error: not enough infor. extraced from (%s). Check ciftokens' % coord) sys.exit() fw.write('\n#\nloop_\n') for p in items: fw.write("%s\n" % p) row = cif.get_rows(items, values) for i in range(len(x)): alter, inst, mod = '.', '.', '1' if model and util.is_number(model[i]): mod = model[i] if alt and alt[i] != '?': alter = alt[i] if ins and ins[i] != '?': inst = ins[i] id1 = '_'.join([mod, comp[i], asym[i], seq[i], alter, inst]) if id1 in compid: xx.append(float(x[i])) yy.append(float(y[i])) zz.append(float(z[i])) for m in row[i]: fw.write("%s " % m) fw.write('\n') else: #pdb format fp = open(coord, 'r') for x1 in fp: if ('CRYST1' in x1[:6]): fw.write(x1) cell = [float(p) for p in x1[8:54].split()] elif ('ATOM' in x1[:4] or 'HETATM' in x1[:6]): alt = x1[16:17] if alt.isspace(): alt = '.' ins = x1[26:27] if ins.isspace(): ins = '.' resname, chid, resnum = x1[17:20].strip(), x1[20:22].strip( ), x1[22:26].strip() resid = '_'.join([resname, chid, resnum, alt, ins]) if resid in compid: fw.write(x1) #only write the selected section xx.append(float(x1[30:38])) yy.append(float(x1[38:46])) zz.append(float(x1[46:54])) fp.close() if not xx or not yy or not zz: print('Error: %s can not be found in the coordinate. try a new id. ' % (compid)) return '', '' frac, orth = util.frac_orth_matrix(cell) #get matrix border = 2.0 xx_min, xx_max = min(xx) - border, max(xx) + border yy_min, yy_max = min(yy) - border, max(yy) + border zz_min, zz_max = min(zz) - border, max(zz) + border xf_min = util.matrix_prod(frac, [xx_min, yy_min, zz_min]) xf_max = util.matrix_prod(frac, [xx_max, yy_max, zz_max]) xyzlim = '%.3f %.3f %.3f %.3f %.3f %.3f' % ( xf_min[0], xf_max[0], xf_min[1], xf_max[1], xf_min[2], xf_max[2]) fw.close() return xyzlim, xyzcomp
def model_vs_data_log(infile,dic1, dic): '''extract values and put them into dic (should be often updated) ''' if not util.check_file(100, infile) : return dic['prog_c'] = 'PHENIX' fr=open(infile, 'r') for ln in fr: if 'Stereochemistry' in ln and 'overall:' in ln: for y in fr: t=y.split(':')[1].split() if 'bonds :' in y: dic['bond'], dic['bond_max'] = t[0], t[1] elif 'angles :' in y: dic['angle'], dic['angle_max'] = t[0], t[1] elif 'dihedrals :' in y: dic['dihedral'], dic['dihedral_max'] = t[0], t[1] elif 'chirality :' in y: dic['chirality'], dic['chirality_max'] = t[0], t[1] elif 'planarity :' in y: dic['planarity'], dic['planarity_max'] = t[0], t[1] elif 'non-bonded (min) :' in y: dic['non_bond'] = t[0].strip() break elif 'Stereochemistry' in ln and 'ligands:' in ln: for y in fr: t=y.split(':')[1].split() if 'bonds :' in y: dic['bond_lig'], dic['bond_lig_max'] = t[0], t[1] elif 'angles :' in y: dic['angle_lig'], dic['angle_lig_max'] = t[0], t[1] elif 'non-bonded (min) :' in y: break elif 'Molprobity statistics:' in ln: for y in fr: if 'outliers :' in y: dic['outlier_num'] = util.str_between_id(y,':', '(') dic['outlier'] = util.str_between_id(y,'(', '%') elif ' allowed :' in y: dic['allowed_num'] = util.str_between_id(y,':', '(') dic['allowed'] = util.str_between_id(y,'(', '%') elif ' favored :' in y: dic['favored_num'] = util.str_between_id(y,':', '(') dic['favored'] = util.str_between_id(y,'(', '%') elif ' Rotamer outliers' in y: dic['rot_outlier_num'] = util.str_between_id(y,':', '(') dic['rot_outlier'] = util.str_between_id(y,'(', '%') elif 'Cbeta deviations' in y: dic['cbeta'] = y.strip().split(':')[1].split()[0] elif 'All-atom clashscore ' in y: dic['clashscore'] = y.strip().split(':')[1].split()[0] elif 'Overall score ' in y: dic['oscore'] = y.strip().split(':')[1].split()[0] break elif 'mFo-DFc map:' in ln: for y in fr: if ' > 3 sigma:' in y : dic['fofc_3sig_p'] = y.strip().split(':')[1] elif ' > 6 sigma:' in y : dic['fofc_6sig_p'] = y.strip().split(':')[1] elif '< -3 sigma:' in y: dic['fofc_3sig_n'] = y.strip().split(':')[1] elif '< -6 sigma:' in y: dic['fofc_6sig_n'] = y.strip().split(':')[1] break elif 'bulk_solvent_(k_sol,b_sol' in ln: tmp=ln.split(':')[1].strip().split() dic1['k_sol'] = tmp[0] dic1['b_sol'] = tmp[1] elif 'high_resolution :' in ln or 'high_resolution :' in ln : dic1['resh']=ln.split(':')[1].strip() elif 'low_resolution :' in ln or 'low_resolution :' in ln : dic1['resl']=ln.split(':')[1].strip() elif 'completeness_in_range' in ln and ':' in ln: tmp=ln.split(':')[1].strip() if(float(tmp)<1.1): dic1['comp']="%.3f" %(float(tmp)*100) else: dic1['comp']=tmp elif 'wilson_b :' in ln or 'wilson_b :' in ln : dic1['bwilson']=ln.split(':')[1].strip() elif ' number_of_reflections :' in ln or 'number_of_reflections :' in ln : dic1['nref']=ln.split(':')[1].strip() elif 'test_set_size :' in ln or 'test_set_size :' in ln: tmp=ln.split(':')[1].strip() if('?' not in dic1['nref']) : dic1['nfree']="%d" %(float(tmp)*float(dic1['nref'])) elif 'twinned :' in ln: twin=ln.split(':')[1].strip() if 'False' not in twin : dic1['twin'] = 'Y' elif 'r_work(re-computed) :' in ln: dic1['rfact']=ln.split(':')[1].strip() elif 'r_free(re-computed) :' in ln: t=ln.split(':')[1].strip() if not util.is_number(t) : t='?' dic1['rfree']=t elif 'TLS : ' in ln: tls_rep=ln.split(':')[1].strip() if 'False' not in tls_rep : dic1['tls_c'] = 'Y' elif 'program_name :' in ln: dic1['prog_r']=ln.split(':')[1].strip() elif 'r_work :' in ln: dic1['rfact_r']=ln.split(':')[1].strip() elif 'r_free :' in ln: dic1['rfree_r']=ln.split(':')[1].strip() elif 'high_resolution :' in ln: dic1['resh_r']=ln.split(':')[1].strip() elif 'low_resolution :' in ln: dic1['resl_r']=ln.split(':')[1].strip() elif 'r_work_cutoff :' in ln: dic1['r_cutoff']=util.float_after_id(ln, ':') elif 'r_free_cutoff :' in ln: dic1['rf_cutoff']=util.float_after_id(ln, ':') elif 'PDB_string' in ln: #for density correlation(atom list) dic1['dcc'] = get_dcc_avg(fr, 1) elif 'resseq resname' in ln: #for density correlation(residue list) dic1['dcc'] = get_dcc_avg(fr, 2) # print(dic['dcc']) fr.close()
def encode(num): if not is_number(num) or num < 0: raise Exception("num is not numeric or < 0!") return _10baseN(num, len(_URL_SAFE_SET), _URL_SAFE_SET)
def update_totalOzone_master_file(self, directory, master_file, date, mode, heading): # noqa """Updates Total Ozone Master File""" # Initialization write_output = 1 current_time = (datetime.now()).strftime("%Y_%m_%d") log_file = open('totalOzone_processing_log_%s' % current_time, 'wb') # noqa data_file = None global tmp_filename tmp_filename = os.path.join(master_file, 'o3tot') if mode == 'overwrite': data_file = open(tmp_filename, 'wb+') else: data_file = open(tmp_filename, 'ab+') if heading == 'on': data_file.write( 'Platform_ID,Year,Month,Day,Start_Hour,Finish_Hour,Wavelength_Pair,Observation_Type,Total_Column_Ozone_Amount,Ozone_Std_Error,Instrument_Type,Instrument_Number\r\n' ) # noqa # external ftp file global output_file output_file = 'Summaries/TotalOzone/Daily_Summary/o3tot.zip' # noqa # extract zipfile zip_flag = False path = directory if zipfile.is_zipfile(directory): zip_flag = True tmpdir = tempfile.mkdtemp() z = zipfile.ZipFile(directory) z.extractall(path=tmpdir) path = tmpdir # traverse the given directory for dirname, dirnames, filenames in os.walk(path): dirnames.sort() filenames.sort() for filename in filenames: try: # print filename file_last_modified_date = time.strftime( "%Y-%m-%d", time.localtime( os.path.getmtime(os.path.join(dirname, filename)))) # noqa # date comparison if date is not None and file_last_modified_date <= date: # noqa log_file.write( 'PROCESSED#%s last modified date: %s\r\n' % ((os.path.join(dirname, filename)), file_last_modified_date)) # noqa extCSV = WOUDCextCSVReader( os.path.join(dirname, filename)) # noqa if date is not None and file_last_modified_date > date: continue if date is None: log_file.write( 'PROCESSED#%s last modified date: %s\r\n' % ((os.path.join(dirname, filename)), file_last_modified_date)) # noqa extCSV = WOUDCextCSVReader( os.path.join(dirname, filename)) # noqa # store data into variables platform_id = ' ' if 'PLATFORM' in extCSV.sections: p_id = extCSV.sections['PLATFORM']['ID'] if p_id is not None and len(p_id) != 0: platform_id = p_id if len(platform_id) == 1: platform_id = '00%s' % platform_id if len(platform_id) == 2: platform_id = '0%s' % platform_id else: log_file.write( 'ERROR#E01:Could not find PLATFORM in input file: %s. Data is ignored\r\n' % os.path.join(dirname, filename)) # noqa inst_type_id = ' ' inst_number = ' 0' if 'INSTRUMENT' in extCSV.sections: inst_name = extCSV.sections['INSTRUMENT']['Name'] inst_model = extCSV.sections['INSTRUMENT']['Model'] if inst_name is not None and len( inst_name) != 0 and inst_name: # noqa try: inst_type_id = util.get_config_value( 'Instrument Type ID', inst_name) # noqa if inst_model == 'Japanese': inst_type_id = util.get_config_value( 'Instrument Type ID', inst_model + ' ' + inst_name) # noqa if len(inst_type_id) == 1: inst_type_id = ' %s' % inst_type_id except Exception, err: log_file.write( 'ERROR#E02:There is no instrumet type id for \'%s\' in file %s. Data is ignored\r\n' % (inst_name, os.path.join( dirname, filename))) # noqa write_output = 0 pass i_num = extCSV.sections['INSTRUMENT']['Number'] if i_num is not None and len(i_num) != 0: inst_number = i_num inst_number = re.sub("^0{1,2}", "", inst_number) # noqa if len(inst_number) == 1: inst_number = ' %s' % inst_number if len(inst_number) == 2: inst_number = ' %s' % inst_number if len(inst_number) == 3: inst_number = ' %s' % inst_number if i_num == 'na': inst_number = ' 0' else: log_file.write( 'ERROR#E03:Could not find INSTRUMENT in input file: %s. Data is ignored\r\n' % os.path.join(dirname, filename)) # noqa if 'DAILY' in extCSV.sections: data = StringIO( (extCSV.sections['DAILY']['_raw']).strip()) # noqa if data is not None: try: data_rows = csv.reader(data) data_rows.next() except StopIteration: log_file.write( 'ERROR#E04:Error reading DAILY block in file %s. Data is ignored\r\n' % os.path.join(dirname, filename)) # noqa write_output = 0 pass for row in data_rows: year = ' ' month = ' ' day = ' ' UTC_Begin = ' ' UTC_End = ' ' WLCode = ' ' ObsCode = ' ' ozone_std_error = ' ' ColumnO3 = ' ' UTC_Mean = ' ' nObs = ' ' if len(row) > 1 and "*" not in row[0]: if len(row[0]) != 0: year = row[0].split('-')[0] month = row[0].split('-')[1] day = row[0].split('-')[2] if len(row) >= 2: if len(row[1]) != 0: WLCode = row[1] if len(WLCode) > 1: try: WLCode = util.get_config_value( 'WLCode', WLCode) # noqa except Exception, err: log_file.write( 'ERROR#E05:There is no one character WLCode code for \'%s\' in file %s. Data is ignored\r\n' % (WLCode, os.path.join( dirname, filename) )) # noqa write_output = 0 pass else: if inst_name == 'Dobson': WLCode = util.get_config_value( 'WLCode', 'Dobson') # noqa if inst_name == 'Brewer': WLCode = util.get_config_value( 'WLCode', 'Brewer') # noqa if inst_name == 'Filter': WLCode = util.get_config_value( 'WLCode', 'Filter') # noqa if inst_name == 'Microtops': WLCode = util.get_config_value( 'WLCode', 'Microtops') # noqa if len(row) >= 3: if len(row[2]) != 0: ObsCode = row[2] if util.is_number( ObsCode) == False and len( ObsCode) != 1: # noqa try: ObsCode = util.get_config_value( 'Obs Code', ObsCode) # noqa except Exception, err: log_file.write( 'ERROR#E06:There is no obs code for \'%s\' in file %s. Data is ignored\r\n' % (ObsCode, os.path.join( dirname, filename) )) # noqa write_output = 0 pass else: ObsCode = '9' if len(row) >= 4: if len( row[3] ) != 0 and row[3] != '0.0' and row[ 3] != '0' and not "-" in row[ 3]: # noqa try: ColumnO3 = '%.0f' % round( float( re.findall( "[0-9]*.[0-9]*", row[3])[0]), 0) # noqa if ColumnO3 == '0': write_output = 0 except Exception, err: log_file.write( 'ERROR#E07:Could not round ColumnO3 value of: %s in file %s. Data ignored.\r\n' % (ColumnO3, os.path.join( dirname, filename))) # noqa write_output = 0 if len(ColumnO3) == 1: ColumnO3 = ' %s' % ColumnO3 # noqa if len(ColumnO3) == 2: ColumnO3 = ' %s' % ColumnO3 # noqa else: write_output = 0 if len(row) >= 6: if len(row[5]) != 0: UTC_Begin = row[5] if len( re.findall( "[0-9]*", UTC_Begin) [0]) > 2: # noqa UTC_Begin = UTC_Begin[:2] elif "-" in UTC_Begin: if -1.5 >= float( UTC_Begin): # noqa UTC_Begin = '-0' else: UTC_Begin = '00' else: try: UTC_Begin = '%.0f' % round( float(UTC_Begin), 0) # noqa except Exception, err: log_file.write( 'ERROR#E08:Could not round UTC_Begin value of: %s in file %s. Data ignored.\r\n' % (UTC_Begin, os.path.join( dirname, filename) )) # noqa write_output = 0 if int(UTC_Begin) in range( 10): # noqa UTC_Begin = '0%s' % UTC_Begin # noqa if len(row) >= 7: if len(row[6]) != 0: UTC_End = row[6] if len( re.findall( "[0-9]*", UTC_End) [0]) > 2: # noqa UTC_End = UTC_End[:2] elif "-" in UTC_End: if -1.5 >= float(UTC_End): UTC_End = '-0' else: UTC_End = '00' else: try: UTC_End = '%.0f' % round( float(UTC_End), 0) # noqa except Exception, err: log_file.write( 'ERROR#E09:Could not round UTC_End value of: %s in file %s. Data ignored.\r\n' % (UTC_End, os.path.join( dirname, filename) )) # noqa write_output = 0 if int(UTC_End) in range( 10): # noqa UTC_End = '0%s' % UTC_End # noqa if len(row) >= 8: if len(row[7]) != 0: UTC_Mean = row[7] if len(row[6]) == 0: UTC_End = UTC_Mean if "-" in UTC_End: if float( UTC_End ) <= -1.5 and float( UTC_End ) > -2: # noqa UTC_End = '-0' elif float( UTC_End ) <= -1 and float( UTC_End ) > -2: # noqa UTC_End = '00' elif float(UTC_End ) >= -1: # noqa UTC_End = '00' elif float( UTC_End ) <= -2 and float( UTC_End ) >= -10: # noqa UTC_End = '-0' elif float( UTC_End ) < -10 and float( UTC_End ) >= -10.5: # noqa UTC_End = '-0' else: UTC_End = '-1' else: try: UTC_End = '%.0f' % round( float(UTC_End), 0) # noqa except Exception, err: log_file.write( 'ERROR#E09:Could not round UTC_End value of: %s in file %s. Data ignored.\r\n' % (UTC_End, os.path.join( dirname, filename)) ) # noqa write_output = 0 if int(UTC_End) in range( 10): # noqa UTC_End = '0%s' % UTC_End # noqa
def process_point(self, p_info, unnamed=True): if isinstance(p_info, str) and not is_number(p_info): assert (Point(p_info) in self.points) return Point(p_info) if not isinstance(p_info, tuple): raise NotImplementedError( f"[process_point] p_info must be tuple or string") p_pred = p_info[0].lower() p_args = p_info[1:] p_val = None if p_pred == "inter-ll": assert (len(p_args) == 2) l1 = self.process_line(p_args[0]) l2 = self.process_line(p_args[1]) p_val = FuncInfo(p_pred, (l1, l2)) elif p_pred in ["isogonal-conj", "isotomic-conj"]: assert (len(p_args) == 4) ps = [self.process_point(p) for p in p_args] p_val = FuncInfo(p_pred, tuple(ps)) elif p_pred == "harmonic-conj": assert (len(p_args) == 3) ps = [self.process_point(p) for p in p_args] p_val = FuncInfo(p_pred, tuple(ps)) elif p_pred in ["incenter", "excenter", "mixtilinear-incenter"]: assert (len(p_args) == 3) ps = [self.process_point(p) for p in p_args] p_val = FuncInfo(p_pred, tuple(ps)) elif p_pred == "inter-lc": assert (len(p_args) == 3) l = self.process_line(p_args[0]) c = self.process_circle(p_args[1]) rs = self.process_rs(p_args[2]) p_val = FuncInfo(p_pred, (l, c, rs)) elif p_pred == "inter-cc": assert (len(p_args) == 3) c1 = self.process_circle(p_args[0]) c2 = self.process_circle(p_args[1]) rs = self.process_rs(p_args[2]) p_val = FuncInfo(p_pred, (c1, c2, rs)) elif p_pred in ["midp", "midp-from"]: assert (len(p_args) == 2) ps = [self.process_point(p) for p in p_args] p_val = FuncInfo(p_pred, tuple(ps)) elif p_pred == "foot": assert (len(p_args) == 2) p = self.process_point(p_args[0]) l = self.process_line(p_args[1]) p_val = FuncInfo(p_pred, (p, l)) elif p_pred == "reflect-pl": assert (len(p_args) == 2) p = self.process_point(p_args[0]) l = self.process_line(p_args[1]) p_val = FuncInfo(p_pred, (p, l)) elif p_pred in ["orthocenter", "circumcenter", "centroid", "incenter"]: assert (len(p_args) == 3) ps = [self.process_point(p) for p in p_args] p_val = FuncInfo(p_pred, tuple(ps)) elif p_pred == "origin": assert (len(p_args) == 1) circ = self.process_circle(p_args[0]) p_val = FuncInfo(p_pred, (circ, )) elif p_pred in ["amidp-opp", "amidp-same"]: assert (len(p_args) == 3) ps = [self.process_point(p) for p in p_args] p_val = FuncInfo(p_pred, tuple(ps)) if p_val is not None: P = Point(p_val) if unnamed: self.unnamed_points.append(P) return P else: raise NotImplementedError( f"[process_point] Unrecognized p_pred {p_pred}")
def main(): # ##### # In Pattern mode, the drone stabilizes above a black pattern that looks like # # # # # # # TX is the wireless camera we were using, and which required special handling: # In TX mode the image is flipped vertically, a small black area near # the borders is removed, and simple de-interlacing is applied. # txMode = False # # DEVO mode is for the older remotes that use a physical Walkera DEVO # drone controller. Leave this off of the small new remotes. # devoMode = False # # If we are running on a respberry computer, # we have the camera interface to grab photos from the RaspberryCam. # berryMode = False # # Images on RaspberryCam are too big, # we need to scale them down. # scaleFactor = 1.0 # create a quad serial object serial = quad_serial.QuadSerial() pid_controllers = [ pid.Pid(0.04, 0, 0), pid.Pid(0.06, 0, 0), pid.Pid(0.04, 0, 0), pid.Pid(0.3, 0, 0) ] coeffs = { "pid": { "xz": { "p": 4, "i": 10, "d": 25 }, "y": { "p": 5, "i": 10, "d": 33 }, "r": { "p": 30, "i": 0, "d": 0 } }, "smoothing": 40 } cameraIndex = 0 # handle arguments for arg in sys.argv[1:]: if arg == "-ngui": viz.GUI.viz_mode = False elif temp == "-pi": berryMode = True elif temp == "-nconfig": viz.GUI.config_mode = False elif util.is_number(temp): cameraIndex = int(temp) viz.GUI.create_trackbars(coeffs) cap = cv.VideoCapture(cameraIndex) frame = np.array([]) camera_matrix = PI_CAMERA_MAT camera_square = np.zeros(3) drone_translation = np.zeros(3) drone_rotation = np.zeros(3) smooth_position = np.zeros(3) last_frame_tick_count = 0 flight_mode_on = False paused = False pressedKey = 0 while pressedKey != 'q': logging.Debug.debug("************** Entered Loop *****************") if not paused: ret, frame = cap.read() if ret is None: break logging.Debug.debug("Photo grabing") if scaleFactor != 1.0: resize(frame, frame, Size(), scaleFactor, scaleFactor, cv.INTER_NEAREST) logging.Debug.debug("Resizing") # TX image processing if txMode: frame = frame[5:, :-7] for i in range(0, frame.rows - 1, 2): frame.row(i + 1).copyTo(frame.row(i)) cv.flip(frame, frame, -1) logging.Debug.debug("TX Mode") viz.GUI.copy_frames(frame) deltaTime = float(cv.getTickCount() - last_frame_tick_count) / cv.getTickFrequency() last_frame_tick_count = cv.getTickCount() logging.Debug.debug("Time since last frame tick count") camera_square = image_processing.ImageProcessing.find_open_square( frame) viz.GUI.draw_square(frame, camera_square) logging.Debug.debug("Tracking OpenSquare section") if camera_square is None: logging.Debug.debug("Square not found") else: retval, rvec, tvec = cv.solvePnP(WORLD_SQUARE, np.float64(camera_square), camera_matrix, 0) logging.Debug.debug("SolvePnP") """ camera_transform = np.concatenate((rvec, tvec.T), axis=1) # The square doesn't move, we are moving. camera_transform = np.linalg.inv(camera_transform) # We found the camera, but we want to find the drone # TODO: Nnot working drone_transform = camera_transform * INV_DRONE_CAMERA_TRANSFORM pos = drone_transform.translation() """ pos = tvec smooth_position = coeffs["smoothing"] * smooth_position + ( 1 - coeffs["smoothing"]) * pos drone_translation = smooth_position drone_rotationd = rvec logging.Debug.debug("found Position") # get latest coeffs from trackbars coeffs = viz.GUI.get_trackbars(coeffs) # update controller coefficients based on trackbar for index, axis in enumerate(["xz", "y", "xz", "r"]): factor = 1000.0 if (axis != "r") else 100.0 pid_controllers[index]._kp = coeffs["pid"][axis]["p"] / factor pid_controllers[index]._ki = coeffs["pid"][axis]["i"] / (factor * 10) pid_controllers[index]._kd = coeffs["pid"][axis]["d"] / (factor * 10) controlErrors = util.calculate_control_errors(drone_translation, drone_rotation, DRONE_TARGET) logging.Debug.debug("PID") # set pid controls pid_control = [0, 0, 0, 0] if camera_square is not None and flight_mode_on: for i in range(4): pid_control.append(pid_controllers[i].calculate( controlErrors[i], deltaTime)) logging.Debug.debug("Take off and Landing") channel_controls = channels.controls_to_channels(pid_control) if not flight_mode_on: channel_controls = [64, 64, 0, 64] if serial.is_opened(): success = serial.send([(int(channel_controls[0]) - 64) * 2, (int(channel_controls[1]) - 64) * 2, (int(channel_controls[2]) - 64) * 2, (int(channel_controls[3]) - 64) * 2]) if not success: logging.Debug.debug("Failed to send to Arduino") logging.Debug.debug("Control errors to console") # Draw GUI viz.GUI.simple_flight_viz(flight_mode_on) """ displayedFrame, displayedFrame, drone_transform, 59.9, droneTarget, pidControl * 100, flight_mode_on """ logging.Debug.debug("Draw GUI") pressedKey = cv.waitKey(0) print type(pressedKey) print pressedKey logging.Debug.debug("cv.waitKey(1)") if pressedKey == ' ': paused = not paused elif pressedKey in [ENTER, PI_ENTER, 'x']: flight_mode_on = not flight_mode_on elif pressedKey in [BACK_BUTTON, PI_BACK_BUTTON, 'i']: for pid_obj in pid_controllers: pid_obj._scaled_error_sum = 0 elif pressedKey in ['q']: break elif pressedKey >= '0' and pressedKey <= '9': viz.GUI.gui_step = pressedKey - '0' logging.Debug.debug("User Commands")
def query_bam(guidesbam, coords, offcoords=True, onebased=False): """Query BAM file containing guideRNA database. Input: guidesbam: path to BAM file coords: genomic coordinates, e.g., str 'chrX:3364088-3372035' or tuple (chr, start, end), e.g., ('chrX', 3364088, 3372035) ; assume coordinates are 0-based offcoords: if True, parse off-target coordinates (takes time), otherwise keep them in int encoded form (can be decoded later with util.map_int_to_coord()); if parsed, coordinates of off-targets are 0-based onebased: if True, input coordinates are 1-based (default is 0-based) Output: list of instances of class GuideRNA """ #create dictionary to verify query coordinates correspond to organism chromosome identity and length #chromosome_dictionary = organism_chromosome_identity_and_length_confirmation(guidesbam) if isinstance(coords, basestring): coords = coords.strip().split(':') coords = (coords[0], ) + tuple(int(x) for x in coords[1].split('-')) chrom, start, end = coords #confirm coordinate exists in genome """ if chromosome_dictionary.has_key(chrom): if start <= chromosome_dictionary[chrom] and end <= chromosome_dictionary[chrom]: pass else: sys.stderr.write('WARNING: %s and/or %s fall off %s which has length %s \n' % (start,end,chrom,chromosome_dictionary[chrom])) return else: sys.stderr.write('WARNING: %s is not a recognized chromosome for this organism \n' % (chrom)) return """ if onebased: start -= 1 # switch to 0-based coordinates, as expected by pysam end -= 1 # and used internally in our pipeline (in the trie etc.) bam = pysam.AlignmentFile(guidesbam, "rb") guides = [] argsdict = ast.literal_eval(bam.header['CO'][3]) genome = argsdict['genome'] delim = util.get_nonexist_int_coord(genome) for r in bam.fetch(chrom, start, end): seq = r.qname positions = r.get_reference_positions() # note: pysam returns 0-based coordinates r_start, r_end = min(positions), max(positions) if r_start < start or r_end > end: # ignore guideRNAs not completely within query genomic region continue strand = '+' if not r.is_reverse else '-' tags = dict(r.get_tags()) offtargets = tags.get('of', []) offdist = tags.get('od', -1) # if 'ds' in r.tags[len(r.tags)-1]: # score = r.tags[len(r.tags)-1][1] score = tags.get('ds', -1) try: if util.is_number(score): score = float(score) else: score = -1 except ValueError: sys.stderr.write( 'ERROR: score %s could not be converted to float, skipping region' % (score)) return if offdist == -1: offdist = argsdict.get('offdist', -1) maxoffcount = tags.get('oc', -1) if maxoffcount == -1: maxoffcount = argsdict.get('maxoffcount', -1) if isinstance(offtargets, basestring): offtargets = util.hex_to_offtargetinfo(offtargets, delim, offcoords) offtargets = [ (p[0], (1 if p[1] == 0 else p[1]) if not offcoords else (1 if p[1][0] == 0 else p[1][0]), [] if not offcoords else [ util.map_int_to_coord( x, genome, onebased=False, strout=False) for x in p[1][1:] ]) for p in offtargets ] guiderna = GuideRNA(seq=seq, coord=(chrom, r_start, r_end, strand), score=score, offdist=offdist, maxoffcount=maxoffcount, offtargets=offtargets) guides.append(guiderna) bam.close() return guides
def reset_tracform(self, src, field=None, author=None, step=0, db=None): """Delete method for all TracForms db tables. Note, that we only delete recorded values and history here, while the form definition (being part of forms parent resource) is retained. Reset of single fields is not implemented, because this would require cumbersome and error prown history rewriting - not worth the hassle. """ db = self._get_db(db) cursor = db.cursor() form_ids = [] # identify form_id(s) to reset if is_number(src): form_ids.append(src) elif isinstance(src, tuple) and len(src) == 3: if src[-1] is None: # no subcontext given, reset all forms of the parent resource for form_id in self.get_tracform_ids(src[0], src[1], db=db): form_ids.append(form_id) else: form_ids.append(self.get_tracform_meta(src, db=db)[0]) db = self._get_db(db) cursor = db.cursor() # restore of old values for multiple forms is not meaningful if step == -1 and len(form_ids) == 1: form_id = form_ids[0] now = int(time.time()) author, updated_on, old_state = self.get_tracform_history( form_id, db=db)[0] or \ (author, now, '{}') if updated_on == now: # no history recorded, so only form values can be reset step = 0 else: # copy last old_state to current cursor.execute( """ UPDATE forms SET author=%s, time=%s, state=%s WHERE id=%s """, (author, updated_on, old_state, form_id)) history = [] records = self.get_tracform_history(form_id, db=db) for history_author, history_time, old_state in records: history.append({ 'author': history_author, 'time': history_time, 'old_state': old_state }) history = parse_history(history, fieldwise=True) # delete restored history entry cursor.execute( """ DELETE FROM forms_history WHERE id=%s AND time=%s """, (form_id, updated_on)) # rollback field info changes for field in history.keys(): changes = history[field] if len(changes) > 0: # restore last field info, unconditional by intention # i.e. to not create entries, if track_fields is False cursor.execute( """ UPDATE forms_fields SET author=%s, time=%s WHERE id=%s AND field=%s """, (changes[0]['author'], changes[0]['time'], form_id, field)) else: # delete current field info cursor.execute( """ DELETE FROM forms_fields WHERE id=%s AND field=%s """, (form_id, field)) if step == 0: # reset all fields and delete full history for form_id in form_ids: cursor.execute( """ DELETE FROM forms_history WHERE id=%s """, (form_id, )) cursor.execute( """ DELETE FROM forms_fields WHERE id=%s """, (form_id, )) # don't delete basic form reference but save the reset # as a form change to prevent creation of a new form_id # for further retention data cursor.execute( """ UPDATE forms SET author=%s, time=%s, state=%s WHERE id=%s """, (author, int(time.time()), '{}', form_id)) db.commit()
break return 9999 while True: offer = telega.last_offer() txt = offer.message bet = txt.split("\n")[-1] if offer.id != last_msg_id: bf = open('bids') for i in bf.read().split("\n"): first_s = i.find(' ') if first_s < 1 or len( i[first_s + 1:]) == 0 or not util.is_number(i[:first_s]): if len(i) > 0: print("Wrong format for bid: " + i) continue if i[first_s + 1:] in txt and init_cost(i) <= int(i[:first_s]): sleep(random.randrange(1, 15)) telega.send_command(bet + "_" + i[:first_s]) bf.close() if last_msg_id != offer.id: print("New offer: " + bet + "; " + txt.split("\n")[1]) last_msg_id = offer.id sleep(1)
def reset_tracform(self, src, field=None, author=None, step=0, db=None): """Delete method for all TracForms db tables. Note, that we only delete recorded values and history here, while the form definition (being part of forms parent resource) is retained. Reset of single fields is not implemented, because this would require cumbersome and error prown history rewriting - not worth the hassle. """ db = self._get_db(db) cursor = db.cursor() form_ids = [] # identify form_id(s) to reset if is_number(src): form_ids.append(src) elif isinstance(src, tuple) and len(src) == 3: if src[-1] is None: # no subcontext given, reset all forms of the parent resource for form_id in self.get_tracform_ids(src[0], src[1], db=db): form_ids.append(form_id) else: form_ids.append(self.get_tracform_meta(src, db=db)[0]) db = self._get_db(db) cursor = db.cursor() # restore of old values for multiple forms is not meaningful if step == -1 and len(form_ids) == 1: form_id = form_ids[0] now = int(time.time()) author, updated_on, old_state = self.get_tracform_history( form_id, db=db)[0] or \ (author, now, '{}') if updated_on == now: # no history recorded, so only form values can be reset step = 0 else: # copy last old_state to current cursor.execute(""" UPDATE forms SET author=%s, time=%s, state=%s WHERE id=%s """, (author, updated_on, old_state, form_id)) history = [] records = self.get_tracform_history(form_id, db=db) for history_author, history_time, old_state in records: history.append({'author': history_author, 'time': history_time, 'old_state': old_state}) history = parse_history(history, fieldwise=True) # delete restored history entry cursor.execute(""" DELETE FROM forms_history WHERE id=%s AND time=%s """, (form_id, updated_on)) # rollback field info changes for field in history.keys(): changes = history[field] if len(changes) > 0: # restore last field info, unconditional by intention # i.e. to not create entries, if track_fields is False cursor.execute(""" UPDATE forms_fields SET author=%s, time=%s WHERE id=%s AND field=%s """, (changes[0]['author'], changes[0]['time'], form_id, field)) else: # delete current field info cursor.execute(""" DELETE FROM forms_fields WHERE id=%s AND field=%s """, (form_id, field)) if step == 0: # reset all fields and delete full history for form_id in form_ids: cursor.execute(""" DELETE FROM forms_history WHERE id=%s """, (form_id,)) cursor.execute(""" DELETE FROM forms_fields WHERE id=%s """, (form_id,)) # don't delete basic form reference but save the reset # as a form change to prevent creation of a new form_id # for further retention data cursor.execute(""" UPDATE forms SET author=%s, time=%s, state=%s WHERE id=%s """, (author, int(time.time()), '{}', form_id)) db.commit()
def model_vs_data_v13_log(infile,dic1, dic): '''extract values and put them into dic (should be often updated) ''' if not util.check_file(100, infile) : return dic['prog_c'] = 'PHENIX' fr=open(infile, 'r') for ln in fr: if ln[0:6] == 'Data: ': for y in fr: t=y.split(':')[1].split() if 'Completeness in resolution range' in y: tmp = t[0].strip() if(float(tmp)<1.1): dic1['comp']="%.3f" %(float(tmp)*100) else: dic1['comp']=tmp elif 'Resolution range:' in y: dic1['resh'] = t[1] dic1['resl'] = t[0] elif 'Number of Miller indices:' in y: dic1['nref'] = t[0] elif 'Wavelength' in y: # Last one break elif ln[0:15] == 'Model vs Data: ': intable = False nnfree = 0 for y in fr: t=y.split(':') if "Resolution Compl Nwork" in y: intable = True continue if intable: t = y.split() if len(t) == 0: if nnfree: dic1['nfree'] = nnfree intable = False continue else: nnfree += int(t[3]) continue if 'r_work:' in y: dic1['rfact'] = t[1].strip() elif 'r_free:' in y: tmp = t[1].strip() if not util.is_number(tmp): tmp='?' dic1['rfree'] = tmp elif 'Number of F-obs outliers' in y: # End of category break elif ln[0:43] == 'Information extracted from PDB file header:': for y in fr: t=y.split(':') if 'program_name' in y: dic1['prog_r'] = t[1].strip() elif 'r_work' in y: dic1['rfact_r'] = t[1].strip() elif 'r_free' in y: dic1['rfree_r'] = t[1].strip() elif 'high_resolution' in y: dic1['resh_r'] = t[1].strip() elif 'low_resolution' in y: dic1['resl_r'] = t[1].strip() elif 'exptl_method' in y: # End of category break elif ln[0:44] == 'After applying resolution and sigma cutoffs:': for y in fr: t=y.split(':') if 'r_work' in y: dic1['r_cutoff'] = t[1].strip() elif 'r_free' in y: dic1['rf_cutoff'] = t[1].strip() elif 'Number of F-obs outliers' in y: # End of category break # print(dic['dcc']) fr.close()
from vocabulary import Vocabulary from util import is_number from majorsystem import MajorSystem if __name__ == "__main__": major_system = MajorSystem() vocab = Vocabulary() prompt = "Enter a number (leave empty and press enter to exit): " number = input(prompt) while number != "": if is_number(number): pattern = major_system.build_pattern(number) words = vocab.search(pattern) print(f'Words for {number}: {words}\n') else: print("That wasn't a valid number.") number = input(prompt)
def xtriage_log(file, dic): ''' parse data from the output of xtriage''' if util.check_file(500, file)==0 : return fp=open(file,'r') dic['twin_ph'] = '?' out1 = [0]*20 spg, twin='?', 'Y' isigi_bin=[] for ln in fp: if 'Space group:' in ln[:12]: spg=ln[12:].split('(')[0].strip() elif 'Resolution range:' in ln[:]: out1[0]=float(ln[17:].split()[1]) elif ' Number of centrics :' in ln[:23]: out1[1]=int(ln[23:].split()[0]) elif ' Number of acentrics :' in ln[:23]: out1[2]=int(ln[23:].split()[0]) elif 'p_value(height) ' in ln and ' : ' in ln : dic['pvalue'] = util.get_value_after_id(ln,":") elif '[MaxAnisoB-MinAnisoB]' in ln[:47]: dic['aniso'] = util.get_value_after_id(ln,":") elif 'max. difference between axes =' in ln: dic['aniso'] = util.get_value_after_id(ln,"=") elif 'The combined Z-score of ' in ln[:25]: t=util.get_value_after_id(ln,"Z-score of") dic['zscore'] = t.split()[0] elif 'No ice ring related problems detected.' in ln: dic['ice_ring'] = '0' elif 'There were ' in ln [:20] and 'ice ring related warnings.' in ln[10:]: dic['ice_ring'] = ln.split()[3] elif ' Patterson function reveals a significant off-origin' in ln: dic['stran'] = 'Y' elif ' No significant pseudotranslation is detected' in ln: dic['stran'] = 'N' elif 'Acentric reflections' in ln: n=0 for tmp in fp: if len(tmp.strip()) <5 : continue n=n+1 if ' <I^2>/<I>^2 :' in tmp: out1[3] = float(tmp[19:].split()[0]) elif ' <F>^2/<F^2> :' in tmp: out1[4] = float(tmp[19:].split()[0]) elif ' <|E^2 - 1|> :' in tmp: out1[5] = float(tmp[19:].split()[0]) elif n>=3: break elif 'Centric reflections' in ln: n=0 for tmp in fp: if len(tmp.strip()) <5 : continue n=n+1 if ' <I^2>/<I>^2 :' in tmp: out1[6] = float(tmp[19:].split()[0]) elif ' <F>^2/<F^2> :' in tmp: out1[7] = float(tmp[19:].split()[0]) elif ' <|E^2 - 1|> :' in tmp: out1[8] = float(tmp[19:].split()[0]) elif n>=3: break elif ' Mean |L| :' in ln[:14]: out1[9]=float(ln[14:].split()[0]) elif ' Mean L^2 :' in ln[:14]: out1[10]=float(ln[14:].split()[0]) elif ' Multivariate Z score L-test:' in ln[:30]: out1[11]=float(ln[30:].split()[0]) elif 'ML estimate of overall B value of ' in ln: n=0 for tmp in fp: n=n+1 if 'A**' in tmp: dic['bwilson']=tmp.split()[0] if(n>=3): dic['bwilson_s']=tmp.split()[0] break elif 'Statistics independent of twin laws' in ln: n=0 for tmp in fp: if len(tmp.strip()) <5 : continue n=n+1 if ' <I^2>/<I>^2 :' in tmp: dic['I2'] = tmp.split(':')[1].strip().split()[0] elif ' <F>^2/<F^2> : ' in tmp: dic['F2'] = tmp.split(':')[1].strip().split()[0] elif ' <|E^2-1|> :' in tmp: dic['E2'] = tmp.split(':')[1].strip().split()[0] elif '<|L|>, <L^2>:' in tmp: ttmp=tmp.split(':')[1] dic['L'] = ttmp.split(',')[0].strip() dic['L2'] = ttmp.split(',')[1].strip() elif '<|L|> :' in tmp: t= tmp.split(':')[1] dic['L'] = t.split()[0] elif '<L^2> :' in tmp: t= tmp.split(':')[1] dic['L2'] = t.split()[0] elif 'Multivariate Z score L-test:' in tmp: dic['Z'] = tmp.split(':')[1].strip() elif n>=6: break elif ('No (pseudo)merohedral twin laws were found' in ln or (' No twinning is suspected.' in ln and util.is_number(dic['twin_r']) and float(dic['twin_r'])>=0.3)) : twin='N' dic['twin_ph'] = 'N' elif '<I/sigma_I>;_Signal_to_noise' in ln[:41]: n=0 for tmp in fp: n=n+1 if n>2 and '$$' in tmp[:4]: break t=tmp.strip().split() if len(t) ==2 : isigi_bin.append(t) elif ' type | R obs. | Britton alpha | H alpha | ML alpha |' in ln: n=0 for tmp in fp: n=n+1 if n>4 : break t=tmp.strip().split('|') if len(t)==8 : dic['twin_op_ph'] = t[1].strip() dic['twin_type'] = t[2].strip() dic['twin_r'] = t[3].strip() dic['twin_fr_ph'] = t[6].strip() if (dic['twin_ph'] != 'N' and util.is_number(dic['twin_r']) and float(dic['twin_r'])<0.3) : dic['twin_ph'] = 'Y' if len(isigi_bin)>3 : dic['i_sigi_resh']=isigi_bin[-1][1] if (util.is_number(isigi_bin[0][1]) and util.is_number(isigi_bin[1][1])): a = float(isigi_bin[0][1])-float(isigi_bin[1][1]) dic['i_sigi_slop'] ='%.1f' %a fp.close() return spg, twin, out1
def __init__(self, n): self.val = n if util.is_number(n): self.val = n.val
def get_list(file): vlist=[] if not util.check_file(200, file): print('Error: file (%s) do not exist' %file) return vlist flist=open(file,'r').readlines() pdbid='XXXX' for x in flist: if 'data_' in x: pdbid=x.split('_')[1].strip() break items,values = cif.cifparse(flist, '_pdbx_density.') sym=cif.parse_values(items,values,'_pdbx_density.space_group_name_H-M') res=cif.parse_values(items,values,'_pdbx_density.ls_d_res_high'); rw=cif.parse_values(items,values,'_pdbx_density.R_value_R_work'); rf=cif.parse_values(items,values,'_pdbx_density.R_value_R_free'); biso=cif.parse_values(items,values,'_pdbx_density.Biso_mean'); bwil=cif.parse_values(items,values,'_pdbx_density.B_wilson'); l2=cif.parse_values(items,values,'_pdbx_density.Padilla-Yeates_L2_mean'); z=cif.parse_values(items,values,'_pdbx_density.Z_score_L_test'); fom=cif.parse_values(items,values,'_pdbx_density.fom'); isig=cif.parse_values(items,values,'_pdbx_density.I_over_sigI_resh'); isigd=cif.parse_values(items,values,'_pdbx_density.I_over_sigI_diff'); pst=cif.parse_values(items,values,'_pdbx_density.translational_pseudo_symmetry'); bsol=cif.parse_values(items,values,'_pdbx_density.B_solvent'); ksol=cif.parse_values(items,values,'_pdbx_density.K_solvent'); tlst=cif.parse_values(items,values,'_pdbx_density.partial_B_value_correction_success'); ntls=cif.parse_values(items,values,'_pdbx_density.tls_group_number'); nncs=cif.parse_values(items,values,'_pdbx_density.ncs_group_number'); nmtx=cif.parse_values(items,values,'_pdbx_density.mtrix_number'); matt=cif.parse_values(items,values,'_pdbx_density.Matthew_coeff'); solv=cif.parse_values(items,values,'_pdbx_density.solvent_content'); dpix=cif.parse_values(items,values,'_pdbx_density.Cruickshank_dpi_xyz'); rtwin=cif.parse_values(items,values,'_pdbx_density.reflns_twin'); xtwin=cif.parse_values(items,values,'_pdbx_density.twin_by_xtriage'); tmp=cif.parse_values(items,values,'_pdbx_density.iso_B_value_type'); if tmp : btype=tmp[0][0] ctwin_t=cif.parse_values(items,values,'_pdbx_density.twin_operator'); ctwin='N' if '2:' in ctwin_t[0] : ctwin='Y' anis=cif.parse_values(items,values,'_pdbx_density.anisotropy'); # looped items,values = cif.cifparse(flist, '_pdbx_density_corr.') prog=cif.parse_values(items,values,'_pdbx_density_corr.program'); resh=cif.parse_values(items,values,'_pdbx_density_corr.ls_d_res_high'); rwork=cif.parse_values(items,values,'_pdbx_density_corr.ls_R_factor_R_work'); rfree=cif.parse_values(items,values,'_pdbx_density_corr.ls_R_factor_R_free'); fcc=cif.parse_values(items,values,'_pdbx_density_corr.correlation_coeff_Fo_to_Fc'); rsr=cif.parse_values(items,values,'_pdbx_density_corr.real_space_R'); dcc=cif.parse_values(items,values,'_pdbx_density_corr.correlation'); detail=cif.parse_values(items,values,'_pdbx_density_corr.details'); nr, nc=0, 0 for i, x in enumerate(detail): if 'Best' in x : nc=i break rprog, cprog = prog[nr].replace(' ', ''), prog[nc] crw, crf, fcc, rsr, dcc=rwork[nc], rfree[nc], fcc[nc],rsr[nc],dcc[nc] rw_crw='?' if util.is_number(rw[0]) and util.is_number(crw): t=int (1000*(float(rw[0]) -float(crw))) rw_crw='%d' %(t) all=[pdbid, res,rw_crw, rw, rf, crw, crf, fcc, rsr, dcc, fom, biso,bwil, matt,solv, ksol, bsol, ntls,nncs, nmtx, tlst,btype, pst, rtwin,xtwin, ctwin, l2, z,anis, isig,isigd, rprog, sym] all_new=[] for x in all: t=x if not x : t='?' else: if (type(x)==list ):t=x[0] y=t.replace(' ', '_') if util.is_number(y) and '.' in y: y='%.2f' %float(y) all_new.append(y) return all_new
def removable_words(words): return [ i for i, w in enumerate(words) if len(w) > 0 and not (is_punctuation(w) or is_number(w)) ]
def write_table(self, path): if not self.page_tables: return # page sort sort_page = [p for p in self.page_tables.keys()] sort_page.sort() # set cell covers i = 1 for page in sort_page: tables = self.page_tables[page] self.page_tables[page] = [] try: for t in tables: info = self.get_table_info(t, page) tt = table_structure.Table(t, self.html) tt.info = info tt.set_table_cells(i) # pdb.set_trace() i += tt.size[0] + 2 self.page_tables[page].append(tt) except: print page print tables # write tables in excel wb = xlwt.Workbook() sheet = wb.add_sheet('tables', cell_overwrite_ok=True) j = 0 for page in sort_page: tables = self.page_tables[page] try: for table in tables: sheet.write(j, 0, "page " + str(page + 1)) sheet.write(j, 1, table.info) j += table.size[0] + 2 for cell in table.cells: if cell.covers: if len(cell.covers) > 1: top_row = cell.covers[0][0] bottom_row = cell.covers[-1][0] left_column = cell.covers[0][1] right_column = cell.covers[-1][1] if util.is_number( cell.text ): # if cell.text is number, save in number style content = cell.text.replace(',', '') if content.isdigit(): sheet.write_merge( top_row, bottom_row, left_column, right_column, float(content)) else: sheet.write_merge( top_row, bottom_row, left_column, right_column, cell.text) else: sheet.write_merge(top_row, bottom_row, left_column, right_column, cell.text) else: if util.is_number(cell.text): content = cell.text.replace(',', '') if content.isdigit(): sheet.write(cell.covers[0][0], cell.covers[0][1], float(content)) else: sheet.write(cell.covers[0][0], cell.covers[0][1], cell.text) else: sheet.write(cell.covers[0][0], cell.covers[0][1], cell.text) except: print page wb.save(path)