def emit(self, do_print, entity_name, indent, reserved_bits, fields, structs, enums, cur_structsize): # embedded structs often won't have a name of their own structname = "a_item" if self.name: structname = "%s_item" % utils.nicename(self.name) bits = self.struct.emit(structname, indent, reserved_bits, fields, structs, enums) # Ignore the condition on some structs' modvals fdesc_id = 0 if len(self.modval) and self.modval.find("=") < 0: fdesc_id = int(self.modval) comment = "" arraybits = "" try: fdesc = fields.get_child(fdesc_id) comment = "\t /* size given by %s */" % utils.nicename(fdesc.name) arraybits = "[0]" except KeyError: pass if do_print: varname = utils.nicename(self.name) if not varname: varname = "item" print "%sstruct %s %s%s;%s" % ("\t" * indent, structname, utils.nicename( self.name), arraybits, comment) return bits
def get_user_id_and_create_if_necessary(self, name): """Returns user ID and create user if necessary. All-in-one wonder method. Searching for the user ID of a given user. If the user does not exist: create the user first and then return the user ID. See also: :meth:`database.create_user`, :meth:`database.get_user_id` Args: name (:obj:`str`): User name. Returns: int: Numeric userID. """ import utils userID = self.get_user_id( utils.nicename(name, self.config['conversion_table'])) if not userID: self.create_user( utils.nicename(name, self.config['conversion_table'])) # - Now loading the newly crates userID userID = self.get_user_id( utils.nicename(name, self.config['conversion_table'])) if not userID: utils.exit( 'OOOH F**K. Created new user for %s but resulting ID was None in get_user_id_and_create_user_if_necessary.' % name) # - Return the id return userID
def emit(self, do_print, entity_name, indent, reserved_bits, fields, structs, enums, cur_structsize): # embedded structs often won't have a name of their own structname = "a_item" if self.name: structname = "%s_item" % utils.nicename(self.name) bits = self.struct.emit(structname, indent, reserved_bits, fields, structs, enums) # Ignore the condition on some structs' modvals fdesc_id = 0 if len(self.modval) and self.modval.find("=") < 0: fdesc_id = int(self.modval) comment = "" arraybits = "" try: fdesc = fields.get_child(fdesc_id) comment = "\t /* size given by %s */" % utils.nicename(fdesc.name) arraybits = "[0]" except KeyError: pass if do_print: varname = utils.nicename(self.name) if not varname: varname = "item" print "%sstruct %s %s%s;%s" % ("\t" * indent, structname, utils.nicename(self.name), arraybits, comment) return bits
def __read_rawfile__(self): """Parsing raw file and import groups and users. """ import re import utils fid = open(self.rawfile,'r') lines = fid.readlines() fid.close() # - Create a dict object and save # groups and users to them. res = {} # - Looping trough lines, search ul found = False group = None for line in lines: if not found and not '<ul type="circle">' in line: continue elif '<ul type="circle">' in line: found = True continue # - if it is a h3 tag, it is a group name. if '<h3' in line: rawgroup = re.sub('<[^<]+?>', '',line).strip() group = utils.nicename( rawgroup ) print ' %-30s %-30s' % (group,rawgroup) res[group] = {'rawgroup':rawgroup,'users':{}} continue # - Reached the end elif '</ul>' in line: break # - Else (if not empty) a user. rawuser = re.sub('<[^<]+?>', '',line).strip() user = utils.nicename( rawuser ) if len(user) == 0: continue else: print ' - %-30s %-30s' % (user,rawuser) res[group]['users'][user] = rawuser print ' Finished ...' self.data = res
def emit(self, do_print, entity_name, indent, reserved_bits, fields, structs, enums, cur_structsize): # modify the field like cProtocolEntityNav::ProcessFragment() num_elements = 0 comment = "" isarray = False if self.modtype == MOD_CONSTANT_ARRAY: # modval is number of elements num_elements = int(self.modval) elif self.modtype == MOD_VARIABLE_ARRAY or \ self.modtype == MOD_VARIABLE_STRING1 or \ self.modtype == MOD_VARIABLE_STRING2 or \ self.modtype == MOD_VARIABLE_STRING3: # The "modval" is the field id that gives the # of elements # of this variable array. That field is usually the immediately # previous fragment of this struct. ie it's something like: # # struct foobar { # u8 arraylen; <- specified by modval # char array[]; <- current struct fragment # } # fdesc = fields.get_child(int(self.modval)) comment = "size given by %s" % utils.nicename(fdesc.name) isarray = True return self.field.emit(do_print, indent, enums, num_elements, comment, isarray)
def emit(self, entity_name, indent, pad_bits, fields, structs, enums): print '%sstruct %s {' % ("\t" * indent, utils.nicename(entity_name)) # current structure size in *bits* size_in_bits = 0 # if there was a previous member we're supposed to add some padding # for (ie, this struct is in a union and something is before it) do # that now if pad_bits: print '%suint8 padding:%d;' % ("\t" * (indent + 1), pad_bits) size_in_bits += pad_bits union_frag = None reserved_bits = 0 for i in range(0, len(self.fragments)): frag = self.fragments[i] if not union_frag: if self.need_union(i): union_frag = frag indent += 1 print '%sunion u_%s {' % ("\t" * indent, utils.nicename(frag.field.name)) bits = frag.emit(True, entity_name, indent + 1, reserved_bits, fields, structs, enums, size_in_bits) if frag == union_frag: # Track the first union fragment's reserved bits; we'll ignore # them for total struct size since the following struct(s) will # include it's size reserved_bits = bits elif is_pad_type(frag.type): # When we reach the pad fragment, the union terminates so # clear out union-specific state reserved_bits = 0 if union_frag: print '%s};' % ("\t" * indent) indent -= 1 union_frag = None size_in_bits += bits else: size_in_bits += bits print "%s};\n" % ("\t" * indent) return size_in_bits
def emit(self, do_print, indent, enums, num_elements, comment, isarray): ctype = '' arraypart = '' sizebits = 0 if self.type == FIELD_TYPE_STD: # eDB2_FIELD_STD tinfo = stdtypes[self.typeval] ctype = tinfo[0] if is_array_type(self.typeval) or num_elements > 0: if num_elements > 0: arraypart = "[%d]" % num_elements sizebits = num_elements * tinfo[2] elif isarray: # array with size given by previous fragment arraypart = "[0]" sizebits = 0 else: arraypart = "[%d]" % (self.size / tinfo[2]) sizebits = self.size * tinfo[2] elif self.type == FIELD_TYPE_ENUM_UNSIGNED or self.type == FIELD_TYPE_ENUM_SIGNED: # It's a enum; find the enum e = enums.get_child(self.typeval) ctype = "gobi_%s" % utils.nicename(e.name) if isarray: if num_elements != 0: raise Exception("Unhandled ENUM field type with size %d" % num_elements) arraypart = "[0]"; sizebits = 0 else: if self.size > 0: # enum size is # of bits arraypart = ":%d" % self.size sizebits = self.size else: sizebits = 32 else: raise ValueError("Unknown Field type") if comment: comment = " (%s)" % comment if do_print: print "%s%s %s%s; /* %s%s */" % ("\t" * indent, ctype, utils.nicename(self.name), arraypart, self.name, comment) return sizebits
def emit(self): print 'typedef enum { /* %s */ ' % self.name for en in self.values: en.emit(self.name) print "} gobi_%s;\n" % utils.nicename(self.name)
def extract_bets(self, block): from pywetterturnier import utils import re if block == 1: data = self.data1 elif block == 2: data = self.data2 else: utils.exit('wrong input to extract_bets. 1 or 2 allowed') # betdate is tdate = self.tdate betdate = tdate + block print '* Extracting data from block %d' % (block) # - Check if cityID is set if self.cityID == None: utils.exit('STOP. cityID equals None. Stop now') # - Loading parameters param = data[1].replace('WvWn', 'Wv Wn').split()[1:] self.unames = [] # - First: find empty line. Those are followed by the # single bets until another empty line shows up or until # the end is reached.. found = False countdown = None for line in data: line = line.replace('\n', '').strip() if "______________" in line: countdown = 3 continue elif not countdown == None: countdown = countdown - 1 if countdown == 0: countdown = None found = True continue elif not found: continue if "ganz nach oben" in line: continue elif len(line.strip()) == 0: continue orig = line.split()[0] username = utils.nicename(orig, self.config['conversion_table']) # - Check if we have found a group if not self.db.get_group_id(username) == False: username = '******' % username # - Now get username userID = self.db.get_user_id_and_create_if_necessary(username) # - Problem detected. if not userID: utils.exit('Problems loading userID for %s' % username) self.unames.append(username) if not len(param) == len(line[len(orig):].split()): utils.exit('parameter length and data length not matching') print ' - %-25s %d' % (username, userID) # - Else write data into database data = line[len(orig):].split() for i in range(0, len(param)): paramID = self.wp_get_param_id(param[i]) if paramID == None: utils.exit('found param which does not exist in db') #print userID,paramID,tdate,betdate,data[i] self.__insert_bet_to_db__(userID, paramID, tdate, betdate, data[i]) self.__insert_betstat_to_db__(userID, tdate) self.db.commit()
def extract_obs(self, block): import re from pywetterturnier import utils if block == 1: data = self.obs1 elif block == 2: data = self.obs2 else: utils.exit('wrong input to extract_obs. 1 or 2 allowed') # betdate is tdate = self.tdate betdate = tdate + block print '* Extracting obs for block %d' % (block) # - Check if cityID is set if self.cityID == None: utils.exit('cityID equals None. Stop now') # - Loading parameters param = data[1].replace('WvWn', 'Wv Wn').split()[1:] # - First: find empty line. Those are followed by the # single bets until another empty line shows up or until # the end is reached.. found = False obscount = 0 print "\n".join(data) for line in data: line = line.replace('\n', '').strip() if "_________________" in line: found = True continue if not found: continue # - Getting station name rawname = line.split()[0].strip() station_name = utils.nicename(rawname, self.config['conversion_table']) wmo = self.__get_wmo_number__(station_name) if not wmo: utils.exit('Cannot find wmo number for station ' + station_name + ' in config file') print ' - Found station ' + str(wmo) + ' ' + station_name # - Check if we do have enough values if not file: if not len(param) == len(line[26:].split()): utils.exit( 'parameter length and data length not matching in obs line' ) obs = line[26:].split() else: # parse that shit by hand obs = [] for i in range(len(param)): obs.append(" ") # now try to read whats here if len(line[len(rawname):]) == 0: continue data = line[len(rawname):].split() if not len(param) == len(data): #utils.exit('noch nicht alle obs da, stop') print 'noch nicht alle obs da, skip' continue for i in range(len(param)): obs[i] = data[i] # - Else write data into database for i in range(0, len(param)): paramID = self.wp_get_param_id(param[i]) if paramID == None: utils.exit('found param which does not exist in db') if len(obs[i].strip()) == 0: continue #####print wmo,paramID,betdate,obs[i] self.__insert_obs_to_db__(wmo, paramID, betdate, obs[i]) # - Increase obscount. If 2: break obscount = obscount + 1 if obscount == 2: break self.db.commit()
def extract_sum_points(self): print '\n * %s' % 'Extracting the sum points for the players' import re # - Cacing results res = [] found = False for line in self.points: psum = None pd1 = None pd2 = None line = line.strip() if 'Name' in line and 'Punkte' in line: continue elif len(line) == 0: continue elif '____________' in line: found = True continue elif not found: continue tmp = line.split() print ' - Found rawname %s: ' % tmp[1] print ' ResT: %s ' % ' '.join(tmp[2:]) rawdata = ''.join(tmp[2:]) rawdata = rawdata.split(')')[0] data = filter(None, re.split('[\(//\)]', rawdata)) if not len(data) == 3 and not len(data) == 1: utils.exit('Cannot split \'%s\' as expected' % rawdata) if len(data) == 3: try: psum = float(data[0]) pd1 = float(data[1]) pd2 = float(data[2]) except: utils.exit('Problems getting points as float from \'%s\'' % rawdata) else: try: psum = float(data[0]) pd1 = False pd2 = False except: utils.exit('Problems getting points as float from \'%s\'' % rawdata) # - Convert raw username (in uname) into nicename nicename = utils.nicename(tmp[1], self.config['conversion_table']) # - Check if we have found a group if not self.db.get_group_id(nicename) == False: nicename = 'GRP_%s' % nicename # - Now get username userID = self.db.get_user_id_and_create_if_necessary(nicename) # - Problem detected. if not userID: utils.exit('Problems loading userID for %s' % nicename) # - Extracting points for which user? print ' Extracting parameter points for: %s' % nicename print ' %7.2f %7.2f %7.2f' % (psum, pd1, pd2) if not pd1: res.append((userID, self.cityID, self.tdate, psum)) else: res.append((userID, self.cityID, self.tdate, psum, pd1, pd2)) if len(res[1]) == 6: #sql = 'INSERT INTO '+self.db.prefix+'wetterturnier_betstat ' + \ sql = 'INSERT INTO '+self.db_betstat+' ' + \ ' (userID, cityID, tdate, points, points_d1, points_d2) VALUES ' + \ ' (%s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE ' + \ ' points=VALUES(points), points_d1=VALUES(points_d1), ' + \ ' points_d2=VALUES(points_d2) ' else: sql = 'INSERT INTO '+self.db_betstat+' ' + \ ' (userID, cityID, tdate, points) VALUES ' + \ ' (%s, %s, %s, %s) ON DUPLICATE KEY UPDATE ' + \ ' points=VALUES(points) ' cur = self.db.cursor() cur.executemany(sql, res) self.db.commit()
def extract_parameter_points(self): if self.unames == None: utils.exit( 'sorry. importbets.unames not set, cannot extract the points') print '\n * %s' % 'Extracting the points for the different parameters now' # - Cacing results res = [] for uname in self.unames: found = False day = None # - Convert raw username (in uname) into nicename nicename = utils.nicename(uname, self.config['conversion_table']) # - Check if we have found a group if not self.db.get_group_id(nicename) == False: nicename = 'GRP_%s' % nicename # - Now get username userID = self.db.get_user_id_and_create_if_necessary(nicename) # - Problem detected. if not userID: utils.exit('Problems loading userID for %s' % nicename) # - Extracting points for which user? print ' - Extracting parameter points for: %s, city %s' % ( nicename, self.config['current_city']) # - Now scanning the lines to get the user points for line in self.raw_lines: # - Check nicename in line if 'Auswertung' in line: if uname == utils.nicename( line.split('Spieler')[1].replace(':','').strip(), self.config['conversion_table'] ) \ or uname.replace('GRP_','') == utils.nicename( line.split('Spieler')[1].replace(':','').strip(), self.config['conversion_table'] ): found = True print ' Found line %s' % line.replace('\n', '').strip() continue else: continue elif not found: continue # - Found a day? if 'Samstag' in line: day = 1 continue elif 'Sonntag' in line: day = 2 continue elif len(line.strip()) == 0: continue elif 'Wert' in line: continue elif '________' in line: continue # - Next Auwertung? JUMP! if 'unktzahl' in line: found = False continue # - Extracting the parameter words = line.split() paramID = self.db.get_parameter_id(words[0]) try: points = float(words[-1]) except: print words if self.file_forced: continue # Log as broken and continue bid = open( self.config['rawdir'] + '/broken_files_identified.txt', 'a+') bid.write("%s\n" % self._file) bid.close() return False #utils.exit('Problems getting points from that line (%s)' % self._file) if not paramID: utils.exit( 'Problems extracting parameter or getting paramID from line' ) #print ' %5d %8.2f %s ' % (paramID,points,line.strip()) # - Results caching. Need # userID form self.db.get_user( uname ) # cityID from self.cityID # paramID from self.db.get_parameter_id( ... ) # tdate from self.tdate # betdate from self.tdate + day res.append( (points, userID,paramID, \ self.tdate,self.tdate + day) ) sql = 'UPDATE '+self.db.prefix+'wetterturnier_bets SET points = %s ' + \ ' WHERE userID = %s AND paramID = %s AND tdate = %s ' + \ ' AND betdate = %s AND cityID = ' + str(self.cityID) cur = self.db.cursor() cur.executemany(sql, res) self.db.commit()
def extract_bettimes(self): import re import datetime as dt from pywetterturnier import utils data = self.bettimes if data == None: print '[!] WARNING bettimes not found. Data empty. Skip.' return # betdate is tdate = self.tdate print '* Extracting bettimes for date %d' % (tdate) # - Check if cityID is set if self.cityID == None: utils.exit('STOP. cityID equals None. Stop now') # - Caching usernames to extract the points afterwards. hopefully. unames = [] # - Going trough lines. # Stop if we find a user without user ID. res = [] found = False for line in data: if "_________________" in line: found = True continue if not found: continue # - Link to jump to the top. Ignore. if "ganz nach oben" in line: continue line = line.replace('\n', '').strip() if len(line) == 0: continue # - Username and its nicename username = line.split()[0].strip() unames.append(username) time = " ".join(line.split()[1:]) # - For Friday, no offset # Oh c'mon wetterturnier scripters. F**k it. if 'xxxxx' in time: tdate_str = dt.datetime.fromtimestamp( 16332 * 86400).strftime('%Y-%m-%d') time = "%s %s" % (tdate_str, '16:00:00') else: if time[0:3] == 'UTC': print 'WRNOG TIME FORMAT, SKIP' continue if 'Fri' in time: tdate_offset = 0 elif 'Mon' in time: tdate_offset = -4 elif 'Tue' in time: tdate_offset = -3 elif 'Wed' in time: tdate_offset = -2 elif 'Thu' in time: tdate_offset = -1 elif 'Sat' in time: tdate_offset = 1 elif 'Sun' in time: tdate_offset = 2 else: print time print line.strip() print '[!] BET TIME FANCY DAY NAME IN DATE. CHEATED? :)' utils.exit('reto think about it') tdate_str = dt.datetime.fromtimestamp( (tdate + tdate_offset) * 86400).strftime('%Y-%m-%d') time = "%s %s" % (tdate_str, time.split()[1]) #time = dt.datetime.strptime(time,'%Y-%m-%d %H:%M:%S').strftime('%s') nicename = utils.nicename(username, self.config['conversion_table']) # - Check if we have found a group if not self.db.get_group_id(nicename) == False: nicename = 'GRP_%s' % nicename # - Now get username userID = self.db.get_user_id_and_create_if_necessary(nicename) # - Problem detected. if not userID: utils.exit('Problems loading userID for %s' % nicename) print ' - [%3d] %-30s %-30s %s' % (userID, username, nicename, time) # - Append res.append((time, userID, tdate, self.cityID)) # - Insert into database print ' - Update database now' cur = self.db.cursor() sql = 'UPDATE '+self.db.prefix+'wetterturnier_bets SET placed = %s ' + \ ' WHERE userID = %s AND tdate = %s AND cityID = %s' cur.executemany(sql, res) self.db.commit() sql = 'UPDATE '+self.db.prefix+'wetterturnier_betstat SET submitted = %s ' + \ ' WHERE userID = %s AND tdate = %s AND cityID = %s' cur.executemany(sql, res) self.db.commit() # - Appending cached raw usernames to the object # This will be used to extract the points afterwards. self.unames = unames