def emit(self, do_print, entity_name, indent, reserved_bits, fields,
structs, enums, cur_structsize):
# embedded structs often won't have a name of their own
structname = "a_item"
if self.name:
structname = "%s_item" % utils.nicename(self.name)
bits = self.struct.emit(structname, indent, reserved_bits, fields,
structs, enums)
# Ignore the condition on some structs' modvals
fdesc_id = 0
if len(self.modval) and self.modval.find("=") < 0:
fdesc_id = int(self.modval)
comment = ""
arraybits = ""
try:
fdesc = fields.get_child(fdesc_id)
comment = "\t /* size given by %s */" % utils.nicename(fdesc.name)
arraybits = "[0]"
except KeyError:
pass
if do_print:
varname = utils.nicename(self.name)
if not varname:
varname = "item"
print "%sstruct %s %s%s;%s" % ("\t" * indent, structname,
utils.nicename(
self.name), arraybits, comment)
return bits
def get_user_id_and_create_if_necessary(self, name):
"""Returns user ID and create user if necessary.
All-in-one wonder method. Searching for the user ID of a given
user. If the user does not exist: create the user first and then
return the user ID.
See also: :meth:`database.create_user`, :meth:`database.get_user_id`
Args:
name (:obj:`str`): User name.
Returns:
int: Numeric userID.
"""
import utils
userID = self.get_user_id(
utils.nicename(name, self.config['conversion_table']))
if not userID:
self.create_user(
utils.nicename(name, self.config['conversion_table']))
# - Now loading the newly crates userID
userID = self.get_user_id(
utils.nicename(name, self.config['conversion_table']))
if not userID:
utils.exit(
'OOOH F**K. Created new user for %s but resulting ID was None in get_user_id_and_create_user_if_necessary.'
% name)
# - Return the id
return userID
def emit(self, do_print, entity_name, indent, reserved_bits, fields, structs, enums, cur_structsize):
# embedded structs often won't have a name of their own
structname = "a_item"
if self.name:
structname = "%s_item" % utils.nicename(self.name)
bits = self.struct.emit(structname, indent, reserved_bits, fields, structs, enums)
# Ignore the condition on some structs' modvals
fdesc_id = 0
if len(self.modval) and self.modval.find("=") < 0:
fdesc_id = int(self.modval)
comment = ""
arraybits = ""
try:
fdesc = fields.get_child(fdesc_id)
comment = "\t /* size given by %s */" % utils.nicename(fdesc.name)
arraybits = "[0]"
except KeyError:
pass
if do_print:
varname = utils.nicename(self.name)
if not varname:
varname = "item"
print "%sstruct %s %s%s;%s" % ("\t" * indent, structname, utils.nicename(self.name), arraybits, comment)
return bits
def __read_rawfile__(self):
"""Parsing raw file and import groups and users.
"""
import re
import utils
fid = open(self.rawfile,'r')
lines = fid.readlines()
fid.close()
# - Create a dict object and save
# groups and users to them.
res = {}
# - Looping trough lines, search ul
found = False
group = None
for line in lines:
if not found and not '<ul type="circle">' in line:
continue
elif '<ul type="circle">' in line:
found = True
continue
# - if it is a h3 tag, it is a group name.
if '<h3' in line:
rawgroup = re.sub('<[^<]+?>', '',line).strip()
group = utils.nicename( rawgroup )
print ' %-30s %-30s' % (group,rawgroup)
res[group] = {'rawgroup':rawgroup,'users':{}}
continue
# - Reached the end
elif '</ul>' in line:
break
# - Else (if not empty) a user.
rawuser = re.sub('<[^<]+?>', '',line).strip()
user = utils.nicename( rawuser )
if len(user) == 0:
continue
else:
print ' - %-30s %-30s' % (user,rawuser)
res[group]['users'][user] = rawuser
print ' Finished ...'
self.data = res
def emit(self, do_print, entity_name, indent, reserved_bits, fields,
structs, enums, cur_structsize):
# modify the field like cProtocolEntityNav::ProcessFragment()
num_elements = 0
comment = ""
isarray = False
if self.modtype == MOD_CONSTANT_ARRAY:
# modval is number of elements
num_elements = int(self.modval)
elif self.modtype == MOD_VARIABLE_ARRAY or \
self.modtype == MOD_VARIABLE_STRING1 or \
self.modtype == MOD_VARIABLE_STRING2 or \
self.modtype == MOD_VARIABLE_STRING3:
# The "modval" is the field id that gives the # of elements
# of this variable array. That field is usually the immediately
# previous fragment of this struct. ie it's something like:
#
# struct foobar {
# u8 arraylen; <- specified by modval
# char array[]; <- current struct fragment
# }
#
fdesc = fields.get_child(int(self.modval))
comment = "size given by %s" % utils.nicename(fdesc.name)
isarray = True
return self.field.emit(do_print, indent, enums, num_elements, comment,
isarray)
def emit(self, do_print, entity_name, indent, reserved_bits, fields, structs, enums, cur_structsize):
# modify the field like cProtocolEntityNav::ProcessFragment()
num_elements = 0
comment = ""
isarray = False
if self.modtype == MOD_CONSTANT_ARRAY:
# modval is number of elements
num_elements = int(self.modval)
elif self.modtype == MOD_VARIABLE_ARRAY or \
self.modtype == MOD_VARIABLE_STRING1 or \
self.modtype == MOD_VARIABLE_STRING2 or \
self.modtype == MOD_VARIABLE_STRING3:
# The "modval" is the field id that gives the # of elements
# of this variable array. That field is usually the immediately
# previous fragment of this struct. ie it's something like:
#
# struct foobar {
# u8 arraylen; <- specified by modval
# char array[]; <- current struct fragment
# }
#
fdesc = fields.get_child(int(self.modval))
comment = "size given by %s" % utils.nicename(fdesc.name)
isarray = True
return self.field.emit(do_print, indent, enums, num_elements, comment, isarray)
def emit(self, entity_name, indent, pad_bits, fields, structs, enums):
print '%sstruct %s {' % ("\t" * indent, utils.nicename(entity_name))
# current structure size in *bits*
size_in_bits = 0
# if there was a previous member we're supposed to add some padding
# for (ie, this struct is in a union and something is before it) do
# that now
if pad_bits:
print '%suint8 padding:%d;' % ("\t" * (indent + 1), pad_bits)
size_in_bits += pad_bits
union_frag = None
reserved_bits = 0
for i in range(0, len(self.fragments)):
frag = self.fragments[i]
if not union_frag:
if self.need_union(i):
union_frag = frag
indent += 1
print '%sunion u_%s {' % ("\t" * indent,
utils.nicename(frag.field.name))
bits = frag.emit(True, entity_name, indent + 1, reserved_bits,
fields, structs, enums, size_in_bits)
if frag == union_frag:
# Track the first union fragment's reserved bits; we'll ignore
# them for total struct size since the following struct(s) will
# include it's size
reserved_bits = bits
elif is_pad_type(frag.type):
# When we reach the pad fragment, the union terminates so
# clear out union-specific state
reserved_bits = 0
if union_frag:
print '%s};' % ("\t" * indent)
indent -= 1
union_frag = None
size_in_bits += bits
else:
size_in_bits += bits
print "%s};\n" % ("\t" * indent)
return size_in_bits
def emit(self, entity_name, indent, pad_bits, fields, structs, enums):
print '%sstruct %s {' % ("\t" * indent, utils.nicename(entity_name))
# current structure size in *bits*
size_in_bits = 0
# if there was a previous member we're supposed to add some padding
# for (ie, this struct is in a union and something is before it) do
# that now
if pad_bits:
print '%suint8 padding:%d;' % ("\t" * (indent + 1), pad_bits)
size_in_bits += pad_bits
union_frag = None
reserved_bits = 0
for i in range(0, len(self.fragments)):
frag = self.fragments[i]
if not union_frag:
if self.need_union(i):
union_frag = frag
indent += 1
print '%sunion u_%s {' % ("\t" * indent, utils.nicename(frag.field.name))
bits = frag.emit(True, entity_name, indent + 1, reserved_bits, fields, structs, enums, size_in_bits)
if frag == union_frag:
# Track the first union fragment's reserved bits; we'll ignore
# them for total struct size since the following struct(s) will
# include it's size
reserved_bits = bits
elif is_pad_type(frag.type):
# When we reach the pad fragment, the union terminates so
# clear out union-specific state
reserved_bits = 0
if union_frag:
print '%s};' % ("\t" * indent)
indent -= 1
union_frag = None
size_in_bits += bits
else:
size_in_bits += bits
print "%s};\n" % ("\t" * indent)
return size_in_bits
def emit(self, do_print, indent, enums, num_elements, comment, isarray):
ctype = ''
arraypart = ''
sizebits = 0
if self.type == FIELD_TYPE_STD: # eDB2_FIELD_STD
tinfo = stdtypes[self.typeval]
ctype = tinfo[0]
if is_array_type(self.typeval) or num_elements > 0:
if num_elements > 0:
arraypart = "[%d]" % num_elements
sizebits = num_elements * tinfo[2]
elif isarray:
# array with size given by previous fragment
arraypart = "[0]"
sizebits = 0
else:
arraypart = "[%d]" % (self.size / tinfo[2])
sizebits = self.size * tinfo[2]
elif self.type == FIELD_TYPE_ENUM_UNSIGNED or self.type == FIELD_TYPE_ENUM_SIGNED:
# It's a enum; find the enum
e = enums.get_child(self.typeval)
ctype = "gobi_%s" % utils.nicename(e.name)
if isarray:
if num_elements != 0:
raise Exception("Unhandled ENUM field type with size %d" % num_elements)
arraypart = "[0]";
sizebits = 0
else:
if self.size > 0:
# enum size is # of bits
arraypart = ":%d" % self.size
sizebits = self.size
else:
sizebits = 32
else:
raise ValueError("Unknown Field type")
if comment:
comment = " (%s)" % comment
if do_print:
print "%s%s %s%s; /* %s%s */" % ("\t" * indent, ctype, utils.nicename(self.name), arraypart, self.name, comment)
return sizebits
def emit(self):
print 'typedef enum { /* %s */ ' % self.name
for en in self.values:
en.emit(self.name)
print "} gobi_%s;\n" % utils.nicename(self.name)
def emit(self):
print 'typedef enum { /* %s */ ' % self.name
for en in self.values:
en.emit(self.name)
print "} gobi_%s;\n" % utils.nicename(self.name)
def extract_bets(self, block):
from pywetterturnier import utils
import re
if block == 1:
data = self.data1
elif block == 2:
data = self.data2
else:
utils.exit('wrong input to extract_bets. 1 or 2 allowed')
# betdate is
tdate = self.tdate
betdate = tdate + block
print '* Extracting data from block %d' % (block)
# - Check if cityID is set
if self.cityID == None:
utils.exit('STOP. cityID equals None. Stop now')
# - Loading parameters
param = data[1].replace('WvWn', 'Wv Wn').split()[1:]
self.unames = []
# - First: find empty line. Those are followed by the
# single bets until another empty line shows up or until
# the end is reached..
found = False
countdown = None
for line in data:
line = line.replace('\n', '').strip()
if "______________" in line:
countdown = 3
continue
elif not countdown == None:
countdown = countdown - 1
if countdown == 0:
countdown = None
found = True
continue
elif not found:
continue
if "ganz nach oben" in line:
continue
elif len(line.strip()) == 0:
continue
orig = line.split()[0]
username = utils.nicename(orig, self.config['conversion_table'])
# - Check if we have found a group
if not self.db.get_group_id(username) == False:
username = '******' % username
# - Now get username
userID = self.db.get_user_id_and_create_if_necessary(username)
# - Problem detected.
if not userID:
utils.exit('Problems loading userID for %s' % username)
self.unames.append(username)
if not len(param) == len(line[len(orig):].split()):
utils.exit('parameter length and data length not matching')
print ' - %-25s %d' % (username, userID)
# - Else write data into database
data = line[len(orig):].split()
for i in range(0, len(param)):
paramID = self.wp_get_param_id(param[i])
if paramID == None:
utils.exit('found param which does not exist in db')
#print userID,paramID,tdate,betdate,data[i]
self.__insert_bet_to_db__(userID, paramID, tdate, betdate,
data[i])
self.__insert_betstat_to_db__(userID, tdate)
self.db.commit()
def extract_obs(self, block):
import re
from pywetterturnier import utils
if block == 1:
data = self.obs1
elif block == 2:
data = self.obs2
else:
utils.exit('wrong input to extract_obs. 1 or 2 allowed')
# betdate is
tdate = self.tdate
betdate = tdate + block
print '* Extracting obs for block %d' % (block)
# - Check if cityID is set
if self.cityID == None:
utils.exit('cityID equals None. Stop now')
# - Loading parameters
param = data[1].replace('WvWn', 'Wv Wn').split()[1:]
# - First: find empty line. Those are followed by the
# single bets until another empty line shows up or until
# the end is reached..
found = False
obscount = 0
print "\n".join(data)
for line in data:
line = line.replace('\n', '').strip()
if "_________________" in line:
found = True
continue
if not found:
continue
# - Getting station name
rawname = line.split()[0].strip()
station_name = utils.nicename(rawname,
self.config['conversion_table'])
wmo = self.__get_wmo_number__(station_name)
if not wmo:
utils.exit('Cannot find wmo number for station ' +
station_name + ' in config file')
print ' - Found station ' + str(wmo) + ' ' + station_name
# - Check if we do have enough values
if not file:
if not len(param) == len(line[26:].split()):
utils.exit(
'parameter length and data length not matching in obs line'
)
obs = line[26:].split()
else:
# parse that shit by hand
obs = []
for i in range(len(param)):
obs.append(" ")
# now try to read whats here
if len(line[len(rawname):]) == 0: continue
data = line[len(rawname):].split()
if not len(param) == len(data):
#utils.exit('noch nicht alle obs da, stop')
print 'noch nicht alle obs da, skip'
continue
for i in range(len(param)):
obs[i] = data[i]
# - Else write data into database
for i in range(0, len(param)):
paramID = self.wp_get_param_id(param[i])
if paramID == None:
utils.exit('found param which does not exist in db')
if len(obs[i].strip()) == 0:
continue
#####print wmo,paramID,betdate,obs[i]
self.__insert_obs_to_db__(wmo, paramID, betdate, obs[i])
# - Increase obscount. If 2: break
obscount = obscount + 1
if obscount == 2:
break
self.db.commit()
def extract_sum_points(self):
print '\n * %s' % 'Extracting the sum points for the players'
import re
# - Cacing results
res = []
found = False
for line in self.points:
psum = None
pd1 = None
pd2 = None
line = line.strip()
if 'Name' in line and 'Punkte' in line:
continue
elif len(line) == 0:
continue
elif '____________' in line:
found = True
continue
elif not found:
continue
tmp = line.split()
print ' - Found rawname %s: ' % tmp[1]
print ' ResT: %s ' % ' '.join(tmp[2:])
rawdata = ''.join(tmp[2:])
rawdata = rawdata.split(')')[0]
data = filter(None, re.split('[\(//\)]', rawdata))
if not len(data) == 3 and not len(data) == 1:
utils.exit('Cannot split \'%s\' as expected' % rawdata)
if len(data) == 3:
try:
psum = float(data[0])
pd1 = float(data[1])
pd2 = float(data[2])
except:
utils.exit('Problems getting points as float from \'%s\'' %
rawdata)
else:
try:
psum = float(data[0])
pd1 = False
pd2 = False
except:
utils.exit('Problems getting points as float from \'%s\'' %
rawdata)
# - Convert raw username (in uname) into nicename
nicename = utils.nicename(tmp[1], self.config['conversion_table'])
# - Check if we have found a group
if not self.db.get_group_id(nicename) == False:
nicename = 'GRP_%s' % nicename
# - Now get username
userID = self.db.get_user_id_and_create_if_necessary(nicename)
# - Problem detected.
if not userID:
utils.exit('Problems loading userID for %s' % nicename)
# - Extracting points for which user?
print ' Extracting parameter points for: %s' % nicename
print ' %7.2f %7.2f %7.2f' % (psum, pd1, pd2)
if not pd1:
res.append((userID, self.cityID, self.tdate, psum))
else:
res.append((userID, self.cityID, self.tdate, psum, pd1, pd2))
if len(res[1]) == 6:
#sql = 'INSERT INTO '+self.db.prefix+'wetterturnier_betstat ' + \
sql = 'INSERT INTO '+self.db_betstat+' ' + \
' (userID, cityID, tdate, points, points_d1, points_d2) VALUES ' + \
' (%s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE ' + \
' points=VALUES(points), points_d1=VALUES(points_d1), ' + \
' points_d2=VALUES(points_d2) '
else:
sql = 'INSERT INTO '+self.db_betstat+' ' + \
' (userID, cityID, tdate, points) VALUES ' + \
' (%s, %s, %s, %s) ON DUPLICATE KEY UPDATE ' + \
' points=VALUES(points) '
cur = self.db.cursor()
cur.executemany(sql, res)
self.db.commit()
def extract_parameter_points(self):
if self.unames == None:
utils.exit(
'sorry. importbets.unames not set, cannot extract the points')
print '\n * %s' % 'Extracting the points for the different parameters now'
# - Cacing results
res = []
for uname in self.unames:
found = False
day = None
# - Convert raw username (in uname) into nicename
nicename = utils.nicename(uname, self.config['conversion_table'])
# - Check if we have found a group
if not self.db.get_group_id(nicename) == False:
nicename = 'GRP_%s' % nicename
# - Now get username
userID = self.db.get_user_id_and_create_if_necessary(nicename)
# - Problem detected.
if not userID:
utils.exit('Problems loading userID for %s' % nicename)
# - Extracting points for which user?
print ' - Extracting parameter points for: %s, city %s' % (
nicename, self.config['current_city'])
# - Now scanning the lines to get the user points
for line in self.raw_lines:
# - Check nicename in line
if 'Auswertung' in line:
if uname == utils.nicename( line.split('Spieler')[1].replace(':','').strip(), self.config['conversion_table'] ) \
or uname.replace('GRP_','') == utils.nicename( line.split('Spieler')[1].replace(':','').strip(), self.config['conversion_table'] ):
found = True
print ' Found line %s' % line.replace('\n',
'').strip()
continue
else:
continue
elif not found:
continue
# - Found a day?
if 'Samstag' in line:
day = 1
continue
elif 'Sonntag' in line:
day = 2
continue
elif len(line.strip()) == 0:
continue
elif 'Wert' in line:
continue
elif '________' in line:
continue
# - Next Auwertung? JUMP!
if 'unktzahl' in line:
found = False
continue
# - Extracting the parameter
words = line.split()
paramID = self.db.get_parameter_id(words[0])
try:
points = float(words[-1])
except:
print words
if self.file_forced: continue
# Log as broken and continue
bid = open(
self.config['rawdir'] + '/broken_files_identified.txt',
'a+')
bid.write("%s\n" % self._file)
bid.close()
return False
#utils.exit('Problems getting points from that line (%s)' % self._file)
if not paramID:
utils.exit(
'Problems extracting parameter or getting paramID from line'
)
#print ' %5d %8.2f %s ' % (paramID,points,line.strip())
# - Results caching. Need
# userID form self.db.get_user( uname )
# cityID from self.cityID
# paramID from self.db.get_parameter_id( ... )
# tdate from self.tdate
# betdate from self.tdate + day
res.append( (points, userID,paramID, \
self.tdate,self.tdate + day) )
sql = 'UPDATE '+self.db.prefix+'wetterturnier_bets SET points = %s ' + \
' WHERE userID = %s AND paramID = %s AND tdate = %s ' + \
' AND betdate = %s AND cityID = ' + str(self.cityID)
cur = self.db.cursor()
cur.executemany(sql, res)
self.db.commit()
def extract_bettimes(self):
import re
import datetime as dt
from pywetterturnier import utils
data = self.bettimes
if data == None:
print '[!] WARNING bettimes not found. Data empty. Skip.'
return
# betdate is
tdate = self.tdate
print '* Extracting bettimes for date %d' % (tdate)
# - Check if cityID is set
if self.cityID == None:
utils.exit('STOP. cityID equals None. Stop now')
# - Caching usernames to extract the points afterwards. hopefully.
unames = []
# - Going trough lines.
# Stop if we find a user without user ID.
res = []
found = False
for line in data:
if "_________________" in line:
found = True
continue
if not found:
continue
# - Link to jump to the top. Ignore.
if "ganz nach oben" in line:
continue
line = line.replace('\n', '').strip()
if len(line) == 0:
continue
# - Username and its nicename
username = line.split()[0].strip()
unames.append(username)
time = " ".join(line.split()[1:])
# - For Friday, no offset
# Oh c'mon wetterturnier scripters. F**k it.
if 'xxxxx' in time:
tdate_str = dt.datetime.fromtimestamp(
16332 * 86400).strftime('%Y-%m-%d')
time = "%s %s" % (tdate_str, '16:00:00')
else:
if time[0:3] == 'UTC':
print 'WRNOG TIME FORMAT, SKIP'
continue
if 'Fri' in time:
tdate_offset = 0
elif 'Mon' in time:
tdate_offset = -4
elif 'Tue' in time:
tdate_offset = -3
elif 'Wed' in time:
tdate_offset = -2
elif 'Thu' in time:
tdate_offset = -1
elif 'Sat' in time:
tdate_offset = 1
elif 'Sun' in time:
tdate_offset = 2
else:
print time
print line.strip()
print '[!] BET TIME FANCY DAY NAME IN DATE. CHEATED? :)'
utils.exit('reto think about it')
tdate_str = dt.datetime.fromtimestamp(
(tdate + tdate_offset) * 86400).strftime('%Y-%m-%d')
time = "%s %s" % (tdate_str, time.split()[1])
#time = dt.datetime.strptime(time,'%Y-%m-%d %H:%M:%S').strftime('%s')
nicename = utils.nicename(username,
self.config['conversion_table'])
# - Check if we have found a group
if not self.db.get_group_id(nicename) == False:
nicename = 'GRP_%s' % nicename
# - Now get username
userID = self.db.get_user_id_and_create_if_necessary(nicename)
# - Problem detected.
if not userID:
utils.exit('Problems loading userID for %s' % nicename)
print ' - [%3d] %-30s %-30s %s' % (userID, username, nicename,
time)
# - Append
res.append((time, userID, tdate, self.cityID))
# - Insert into database
print ' - Update database now'
cur = self.db.cursor()
sql = 'UPDATE '+self.db.prefix+'wetterturnier_bets SET placed = %s ' + \
' WHERE userID = %s AND tdate = %s AND cityID = %s'
cur.executemany(sql, res)
self.db.commit()
sql = 'UPDATE '+self.db.prefix+'wetterturnier_betstat SET submitted = %s ' + \
' WHERE userID = %s AND tdate = %s AND cityID = %s'
cur.executemany(sql, res)
self.db.commit()
# - Appending cached raw usernames to the object
# This will be used to extract the points afterwards.
self.unames = unames
