def package_type(ini_parser, tar, tar_path, files):
if not ini_parser.has_option("musa", "type"):
raise MusaException("no type specified in the configuration file")
type_name = ini_parser.get("musa", "type")
if not type_name in types:
raise MusaException("unknown type \"%s\"" % type_name)
metadata = {'package_type': type_name}
# uniqueid and content title cannot be obtained from the Scenario (.scn) or Heightmap image file.
# So for them this need to be obtained from the .ini file instead and passed to their package
# methods.
if type_name in ["Scenario", "Heightmap"]:
uniqueid_str = ini_parser.get("musa", "uniqueid")
if len(uniqueid_str) == 8:
uniqueid = int(uniqueid_str, 16)
elif (len(uniqueid_str) > 0) and uniqueid_str[0] != '0':
uniqueid = int(uniqueid_str, 10)
else:
raise MusaException("Invalid uniqueid syntax in ini file")
title = get_scen_hm_title(ini_parser.get("musa", "name"),
ini_parser.get("musa", "version"))
metadata.update(types[type_name]['package'](tar, tar_path, files,
uniqueid, title))
else:
metadata.update(types[type_name]['package'](tar, tar_path, files))
return metadata
def package_text(ini_parser, tar, tar_path, package_files):
if ini_parser.has_option("musa", "description_file"):
description_file = ini_parser.get("musa", "description_file")
try:
with open(description_file, 'r') as content_file:
description_text = content_file.read()
except:
raise MusaException("unknown description file \"%s\"" % description_file)
elif ini_parser.has_option("musa", "description_text"):
description_text = ini_parser.get("musa", "description_text")
else:
raise MusaException("Neither description_text nor description_file specified in the configuration file")
if len(description_text) > 512:
raise MusaException("value for description is too long")
check_utf8(description_text, "description")
for fname in list(package_files):
bname = os.path.basename(fname)
if docs_regexp.match(bname):
with open(fname, 'r') as content:
check_utf8(content.read(), bname)
tar.add(fname, arcname=os.path.join(tar_path, bname))
package_files.remove(fname)
return { 'description': description_text }
def package_misc(ini_parser):
metadata = {}
for field in fields.keys():
if not ini_parser.has_option("musa", field):
raise MusaException("no %s specified in the configuration file" % field)
value = ini_parser.get("musa", field)
if len(value) > fields[field]:
raise MusaException("value for %s is too long" % field)
check_utf8(value, field)
metadata[field] = value
metadata['min_version'] = parse_version(ini_parser, 'openttd_minimum_supported_version', 0x06000000)
metadata['max_version'] = parse_version(ini_parser, 'openttd_maximum_supported_version', -1)
metadata['safe_name'] = safe_filename(metadata["name"]) + "-" + safe_filename(metadata["version"], True)
metadata['tags'] = parse_list(ini_parser, 'tags')
for tag in metadata['tags']:
if len(tag) > 32:
raise MusaException("invalid tag name")
metadata['authors'] = parse_list(ini_parser, 'authors')
raw_deps = parse_list(ini_parser, 'dependencies')
dep_list = list()
for dep in raw_deps:
type, uniqueid, md5sum = checkdependency(dep)
dep_list.append(type + ":" + "%08X" % uniqueid + ":" + "%032x" % md5sum)
metadata['dependencies'] = dep_list
return metadata
def validate_script(metadata, tar, tar_path, suspect_filenames, infofile):
if tar is None:
return
filename = find_exact_file_in_list(tar.getnames(), infofile)
short_name = get_script_short_name(tar.extractfile(filename))
if len(short_name) != 4:
raise MusaException("Invalid short name")
uniqueid = (ord(short_name[0]) << 0) + (ord(short_name[1]) << 8) + (
ord(short_name[2]) << 16) + (ord(short_name[3]) << 24)
if metadata['uniqueid'] != uniqueid:
raise MusaException("Invalid short name")
md5sum = 0
for file in tar.getnames():
ext = file.split(".")[-1]
if ext == "nut":
md5sum ^= validate_md5(tar.extractfile(file), None,
tar.getmember(file).size)
if file in suspect_filenames:
if ext == "nut" or re.search("\/|\\\\lang\/|\\\\.*\.txt$",
file.lower()) != None:
suspect_filenames.remove(file)
if metadata['md5sum'] != "%032x" % md5sum:
raise MusaException("md5sum mismatch")
def validate_bg(metadata, tar, tar_path, suspect_filenames):
if tar is None:
return
(filename, ini_parser,
shortname) = validate_packaged_ini(tar, obg_ini, ".obg")
if os.path.dirname(filename) != tar_path:
raise MuseException("obg file %s in wrong folder" % fname)
suspect_filenames.remove(filename)
if metadata['uniqueid'] != (ord(shortname[0]) << 0) + (
ord(shortname[1]) << 8) + (ord(shortname[2]) << 16) + (
ord(shortname[3]) << 24):
raise MusaException("uniqueid mismatch")
grfs = get_md5sums(ini_parser, obg_ini, tar.getnames())
md5sum = 0
for name, fname, md5 in grfs:
if os.path.dirname(fname) != tar_path:
raise MuseException("grf file %s in wrong folder" % fname)
md5sum ^= validate_md5(tar.extractfile(fname), md5,
tar.getmember(fname).size)
suspect_filenames.remove(fname)
if metadata['md5sum'] != "%032x" % md5sum:
raise MusaException("md5sum mismatch")
def check_utf8(text, type):
if not isinstance(text, str):
raise MusaException("%s is not a string" % type)
try:
text.decode('utf-8')
except UnicodeDecodeError:
raise MusaException("%s is not a valid UTF-8 content" % type)
def find_exact_file_in_list(files, filename):
rets = [file for file in files if os.path.basename(file) == filename]
if len(rets) > 1:
raise MusaException("multiple %s files" % filename)
if len(rets) == 0:
raise MusaException("no %s file" % filename)
return rets[0]
def handle_version(self, version):
if not isinstance(version, str):
raise MusaException("invalid type for version")
if version != "0.0.0":
raise MusaException("please update your client to version %s" %
version)
self.state = self.handle_authorization
def find_file_in_list(files, end):
rets = []
for e in end:
rets += [file for file in files if file.endswith(e)]
if len(rets) > 1:
raise MusaException("multiple %s files" % end)
if len(rets) == 0:
raise MusaException("no %s file" % end)
return rets[0]
def validate_newgrf(metadata, tar, tar_path, suspect_filenames):
if metadata['uniqueid'] >> 24 == 0xFF:
raise MusaException("Invalid/system GRF")
if tar is None:
return
filename = find_file_in_list(tar.getnames(), [".grf"])
md5sum = validate_md5(tar.extractfile(filename), metadata['md5sum'],
tar.getmember(filename).size)
if metadata['uniqueid'] != get_grfid(tar.extractfile(filename)):
raise MusaException("uniqueid mismatch")
suspect_filenames.remove(filename)
def validate_ini(file, expected_sections):
ini_parser = ConfigParser()
ini_parser.readfp(file)
for section, keys in obg_ini.items():
if not ini_parser.has_section(section):
raise MusaException("section %s is missing" % section)
for key in keys:
if not ini_parser.has_option(section, key):
raise MusaException("option %s:%s is missing" % (section, key))
shortname = ini_parser.get("metadata", "shortname")
if len(shortname) != 4:
raise MusaException("the short name is not 4 long")
return (ini_parser, shortname)
def package_script(tar, tar_path, files, infofile):
filename = find_exact_file_in_list(files, infofile)
short_name = get_script_short_name(open(filename))
if len(short_name) != 4:
raise MusaException("Invalid short name")
uniqueid = (ord(short_name[0]) << 0) + (ord(short_name[1]) << 8) + (
ord(short_name[2]) << 16) + (ord(short_name[3]) << 24)
md5sum = 0
scriptFiles = list()
for file in files:
ext = file.split(".")[-1]
if ext == "nut":
md5sum ^= validate_md5(open(file, "rb"), None,
os.stat(file).st_size)
if ext == "nut" or re.search("\/|\\\\lang\/|\\\\.*\.txt$",
file) != None:
scriptFiles.append(file)
common_path = os.path.commonprefix(scriptFiles)
for file in scriptFiles:
content_file_path = file[len(common_path):].replace(
"\\", "/") # remove common path
tar.add(file, arcname=os.path.join(tar_path, content_file_path))
files.remove(file)
return {'uniqueid': uniqueid, 'md5sum': "%032x" % md5sum}
def validate_tar_file(tar, tar_path, filename, expected_content):
tar_file = tar_join_path(tar_path, filename)
f_expected = StringIO.StringIO(expected_content)
f_expected.seek(0, os.SEEK_END)
expected_size = f_expected.tell()
try:
f = tar.extractfile(tar_file)
except tarfile.TarError:
raise MusaException("Tar does not contain expected file " + filename)
if tar.getmember(tar_file).size != expected_size:
raise MusaException("File " + filename +
" in tar does not has the expected size")
if f.read(expected_size) != expected_content:
raise MusaException("File " + filename +
" in tar does not contain the expected content")
def authenticate(username, password):
username = username.encode('utf-8')
password = password.encode('utf-8')
user_dn = "uid=" + username.lower() + ",ou=Users,dc=openttd,dc=org"
ldap_conn = ldap.initialize(LDAP_HOST)
try:
ldap_conn.bind_s(user_dn, password)
except ldap.INVALID_CREDENTIALS:
raise MusaException("Invalid username/password")
r = ldap_conn.search_s('ou=Users,dc=openttd,dc=org', ldap.SCOPE_SUBTREE,
"(uid=" + username.lower() + ")", ["memberOf"])
if not 'cn=BaNaNaS-Manager,ou=Manager,ou=Groups,dc=openttd,dc=org' in r[0][
1]['memberOf']:
raise MusaException("You are no bananas manager, you may not upload")
return True
def handle_metadata(self, metadata):
if not isinstance(metadata, dict):
raise MusaException("invalid type for metadata")
self.metadata = metadata
validate(self.metadata, None)
if not self.user in self.metadata['authors']:
raise MusaException(
"you are not listed as author for this content")
self.db_conn = _mysql.connect(DATABASE_HOST, DATABASE_USER,
DATABASE_PASSWORD, DATABASE_NAME)
self.db_conn.autocommit(False)
check_content(self.user, self.metadata, self.db_conn)
self.send("metadata validated at server side")
self.binary = True
self.state = self.handle_upload
def get_grfid(f):
try:
reader = GRFIDReader(f)
grfcontversion = 1
# Check version
if reader.buffer[0:len(grfv2header)] == grfv2header:
grfcontversion = 2
reader.skip_bytes(len(grfv2header) + 4 + 1)
if reader.read_size(
grfcontversion) != 0x04 or reader.read_byte() != 0xFF:
raise MusaException("No magic header")
reader.read_dword()
while True:
num = reader.read_size(grfcontversion)
if num == 0:
# End of file, but no GRFID
raise MusaException("No GRFID")
type = reader.read_byte()
if type == 0xFF:
action = reader.read_byte()
if action == 0x08:
# Ignored version
reader.read_byte()
# Finally... the GRFID
return reader.swap(reader.read_dword())
else:
# Skip pseudo sprites
reader.skip_bytes(num - 1)
else:
# Skip real sprites
reader.skip_bytes(7)
reader.skip_sprite_data(type, num - 8)
except Exception, inst:
f.close()
raise inst
def checkdependency(dep):
data = dep.split(":")
if len(data) != 3:
raise MusaException("invalid dependency")
if not data[0] in [ "AI", "AI Library", "Base Graphic", "Base Music", "Base Sound", "Game Script", "GS Library", "Heightmap", "NewGRF", "Scenario" ]:
raise MusaException("invalid dependency type")
if len(data[1]) == 4:
short_name = data[1]
id = (ord(short_name[0]) << 0) + (ord(short_name[1]) << 8) + (ord(short_name[2]) << 16) + (ord(short_name[3]) << 24)
elif len(data[1]) == 8:
try:
id = int(data[1], 16)
except:
raise MusaException("invalid dependency id")
else:
raise MusaException("invalid dependency id")
if len(data[2]) != 32:
raise MusaException("invalid dependency md5sum")
try:
md5 = int(data[2], 16)
except:
raise MusaException("invalid dependency id")
return (data[0], id, md5)
def parse_list(ini_parser, field):
if not ini_parser.has_option("musa", field):
raise MusaException("no %s specified in the configuration file" % field)
ret = set()
data = ini_parser.get("musa", field)
if len(data) != 0:
for value in data.split(","):
value = value.strip()
check_utf8(value, field)
ret.add(value)
return list(ret)
def package_newgrf(tar, tar_path, files):
filename = find_file_in_list(files, [".grf"])
uniqueid = get_grfid(open(filename, 'rb'))
if uniqueid >> 24 == 0xFF:
raise MusaException("Invalid/system GRF")
md5sum = validate_md5(open(filename, "rb"), None,
os.stat(filename).st_size)
tar.add(filename,
arcname=os.path.join(tar_path, os.path.basename(filename)))
files.remove(filename)
return {'uniqueid': uniqueid, 'md5sum': "%032x" % md5sum}
def get_md5sums(ini_parser, sections, files):
grfs = []
for key in sections['files']:
name = ini_parser.get('files', key)
if not ini_parser.has_option('md5s', name):
raise MusaException("no MD5 checksum given for %s" % name)
md5 = ini_parser.get('md5s', name)
fname = find_file_in_list(files, [name])
grfs.append((name, fname, md5))
return grfs
def get_script_type(f):
for line in f.readlines():
if re.search("extends\\s+GSInfo", line) != None:
return 'GS'
elif re.search("extends\\s+GSLibrary", line) != None:
return 'GS'
elif re.search("extends\\s+AIInfo", line) != None:
return 'AI'
elif re.search("extends\\s+AILibrary", line) != None:
return 'AI'
raise MusaException('Unknown script type')
def validate(metadata, tar, verbose=False):
if not 'safe_name' in metadata:
raise MusaException("safe name is missing")
if not isinstance(metadata['safe_name'], str):
raise MusaException("safe name is invalid")
tar_path = metadata['safe_name']
if tar is not None:
suspect_filenames = tar.getnames()
for fn in suspect_filenames:
if not fn.startswith(tar_path):
raise MusaException("invalid file in tarball")
if tar.getmember(fn).isdir():
suspect_filenames.remove(fn)
else:
suspect_filenames = []
if verbose: print "validating misc data"
validate_misc(metadata, tar, tar_path, suspect_filenames)
if verbose: print "validating license..."
validate_license(metadata, tar, tar_path, suspect_filenames)
if verbose: print "validating text"
validate_text(metadata, tar, tar_path, suspect_filenames)
if verbose: print "validating type..."
validate_type(metadata, tar, tar_path, suspect_filenames)
if len(suspect_filenames) != 0:
if verbose:
print "the following unknown files are found:"
for sf in suspect_filenames:
print " - %s" % sf
raise MusaException("unknown files in tarball")
def validate_md5(file, md5, size):
try:
md5sum = hashlib.md5()
read_header = file.read(10)
md5sum.update(read_header)
read_header = [ord(i) for i in read_header]
size -= 10
expected_header = list(
array.array('B', [
0x00, 0x00,
ord('G'),
ord('R'),
ord('F'), 0x82, 0x0D, 0x0A, 0x1A, 0x0A
]))
if read_header == expected_header:
raw_size = file.read(4)
size -= 4
md5sum.update(raw_size)
size = unpack("<i", raw_size)[0]
while size > 0:
data = file.read(min(8192, size))
if data is None or len(data) == 0:
break
md5sum.update(data)
size -= len(data)
digest = md5sum.hexdigest()
if md5 is not None and digest != md5:
print digest
print md5
raise MusaException("MD5 checksums do not match!")
return int(digest, 16)
except:
file.close()
raise
def validate_type(metadata, tar, tar_path, suspect_filenames):
if not 'package_type' in metadata:
raise MusaException("package type is missing")
if not isinstance(metadata['package_type'], str):
raise MusaException("package_type is invalid")
package_type = metadata['package_type']
if not package_type in types:
raise MusaException("invalid package type")
if not "uniqueid" in metadata:
raise MusaException("uniqueid missing")
if not isinstance(metadata['uniqueid'], int):
raise MusaException("uniqueid is invalid")
if not "md5sum" in metadata:
raise MusaException("md5sum missing")
if not isinstance(metadata['md5sum'], str):
raise MusaException("md5sum is invalid")
types[package_type]['validate'](metadata, tar, tar_path, suspect_filenames)
def handle_authorization(self, info):
if not isinstance(info, dict):
raise MusaException("invalid type for info")
if not "username" in info or not isinstance(info["username"], str):
raise MusaException("invalid type for info")
if not "password" in info or not isinstance(info["password"], str):
raise MusaException("invalid type for info")
if not "check" in info or not isinstance(info["check"], str):
raise MusaException("invalid type for info")
if info["check"] != "yes I am":
raise MusaException(
"you are not the author of this content. You may not upload it"
)
if not authenticate(info["username"], info["password"]):
raise MusaException("could not authenticate")
self.user = info["username"]
self.state = self.handle_metadata
def parse_version(ini_parser, name, default):
if not ini_parser.has_option("musa", name):
raise MusaException("no %s specified in the configuration file" % name)
version = ini_parser.get("musa", name)
if len(version) == 0:
return default
version = version.split(" ", 2)
if len(version) == 1:
version = version[0].split(".")
if len(version) != 3:
raise MusaException("invalid full version")
iversion = []
try:
for v in version:
iversion.append(int(v))
except:
raise MusaException("invalid full version")
if name == 'openttd_maximum_supported_version':
stable_comp = 0x0008FFFF
else:
stable_comp = 0x00080000
return iversion[0] * 0x10000000 + iversion[1] * 0x01000000 + iversion[2] * 0x00100000 + stable_comp
else:
if not version[1].startswith("r"):
raise MusaException("invalid revision")
try:
revision = int(version[1][1:])
version = version[0].split(".")
if len(version) != 2:
raise MusaException("invalid full version")
iversion = []
for v in version:
iversion.append(int(v))
except:
raise MusaException("invalid full version")
return iversion[0] * 0x10000000 + iversion[1] * 0x01000000 + revision
def validate_text(metadata, tar, tar_path, suspect_filenames):
if not 'description' in metadata:
raise MusaException("description is missing")
if not isinstance(metadata['description'], str):
raise MusaException("description is invalid")
if len(metadata['description']) > 512:
raise MusaException("value for description is too long")
check_utf8(metadata['description'], "description")
if not 'name' in metadata:
raise MusaException("name is missing")
if not isinstance(metadata['name'], str):
raise MusaException("name is invalid")
if len(metadata['name']) > 32:
raise MusaException("value for name is too long")
check_utf8(metadata['name'], "name")
if tar is not None:
for member in tar.getnames():
bname = os.path.basename(member)
if docs_regexp.match(bname):
try:
fd = tar.extractfile(member)
check_utf8(fd.read(), bname)
fd.close()
except:
fd.close()
raise
if os.path.dirname(member) != tar_path:
raise MuseException("text file %s in wrong folder" % member)
suspect_filenames.remove(member)
def fill_buffer(self):
self.buffer = [ord(i) for i in self.file.read(8192)]
if len(self.buffer) == 0:
raise MusaException("Invalid GRF")
def validate_misc(metadata, tar, tar_path, suspect_filenames):
for field in fields.keys():
if not field in metadata:
raise MusaException("%s missing" % field)
value = metadata[field]
if len(value) > fields[field]:
raise MusaException("value for %s is too long" % field)
check_utf8(value, field)
if not 'safe_name' in metadata:
raise MusaException("safe_name missing")
if not isinstance(metadata['safe_name'], str):
raise MusaException("safe name is invalid")
if not 'min_version' in metadata:
raise MusaException("min_version missing")
if not isinstance(metadata['min_version'], int):
raise MusaException("min_version is invalid")
if not 'max_version' in metadata:
raise MusaException("min_version missing")
if not isinstance(metadata['max_version'], int):
raise MusaException("max_version is invalid")
min_version = int(metadata['min_version'])
max_version = int(metadata['max_version'])
if max_version != -1 and min_version > max_version or min_version < 0 or max_version < -1:
raise MusaException("invalid version")
if metadata['safe_name'] != safe_filename(metadata["name"]) + "-" + safe_filename(metadata["version"], True):
raise MusaException("safe_name inconsistent")
if not 'tags' in metadata:
raise MusaException("tags missing")
if not isinstance(metadata['tags'], list):
raise MusaException("tags is invalid")
for tag in metadata['tags']:
check_utf8(tag, "tags")
if len(tag) > 32:
raise MusaException("invalid tag name")
if not 'authors' in metadata:
raise MusaException("authors missing")
if not isinstance(metadata['authors'], list):
raise MusaException("authors is invalid")
if len(metadata['authors']) < 1:
raise MusaException("authors missing")
for author in metadata['authors']:
check_utf8(author, "authors")
if not 'dependencies' in metadata:
raise MusaException("dependencies missing")
if not isinstance(metadata['dependencies'], list):
raise MusaException("dependencies is invalid")
for dep in metadata['dependencies']:
check_utf8(dep, "dependencies")
checkdependency(dep)
def check_content(username, metadata, db_conn):
# Verify name
db_conn.query("""
SELECT username
FROM bananas_file
JOIN bananas_type ON bananas_file.type_id = bananas_type.id
JOIN bananas_file_authors ON bananas_file.id = bananas_file_authors.file_id
JOIN bananas_author ON bananas_author.id = bananas_file_authors.author_id
JOIN auth_user ON auth_user.id = bananas_author.user_id
WHERE bananas_file.name = "%s" AND bananas_type.name = "%s"
ORDER BY bananas_file.id
""" % (_mysql.escape_string(metadata['name']), metadata['package_type']))
users = []
for row in db_conn.store_result().fetch_row(maxrows=0):
users.append(row[0])
if len(users) > 0 and not username in users:
raise MusaException(
"the supplied content name is already used by another author")
# Verify ownership of uniqueid
db_conn.query("""
SELECT bananas_file.id, filename, version, uniquemd5, blacklist, published, username
FROM bananas_file
JOIN bananas_type ON bananas_file.type_id = bananas_type.id
JOIN bananas_file_authors ON bananas_file.id = bananas_file_authors.file_id
JOIN bananas_author ON bananas_author.id = bananas_file_authors.author_id
JOIN auth_user ON auth_user.id = bananas_author.user_id
WHERE uniqueid = %d AND bananas_type.name = "%s"
ORDER BY bananas_file.id
""" % (metadata['uniqueid'], metadata['package_type']))
prevId = None
users = []
for row in db_conn.store_result().fetch_row(maxrows=0):
if row[1] == metadata['safe_name']:
raise MusaException("this content is already uploaded (filename)")
if row[2] == metadata['version']:
raise MusaException("this content is already uploaded (version)")
if row[3] == metadata['md5sum']:
raise MusaException("this content is already uploaded (md5)")
if row[4] == "1":
raise MusaException("this content is blacklisted")
if row[5] == "1":
prevId = int(row[0])
users.append(row[6])
if len(users) > 0 and username not in users:
raise MusaException(
"you are no author for this content; you cannot update it")
if prevId == None and metadata['package_type'] in [
'Scenario', 'Heightmap'
]:
raise MusaException(
"heightmaps and scenarios must be initially uploaded via bananas web manager to obtain a uniqueid"
)
metadata['prevId'] = prevId
metadata['resolved_dependencies'] = []
for dep in metadata['dependencies']:
act_dep = dep.split(":")
db_conn.query("""
SELECT bananas_file.id FROM bananas_file
JOIN bananas_type ON bananas_file.type_id = bananas_type.id
WHERE uniqueid = %d AND bananas_type.name = "%s" AND uniquemd5 = "%s"
""" % (int(act_dep[1], 16), act_dep[0], act_dep[2]))
data = db_conn.store_result().fetch_row(maxrows=0)
if len(data) == 0:
raise MusaException("dependency %s not in bananas" % dep)
if len(data) > 1:
raise MusaException("duplicate %s dependencies in bananas" % dep)
metadata['resolved_dependencies'].append(int(data[0][0]))
metadata['resolved_authors'] = []
for author in metadata['authors']:
db_conn.query("""
SELECT bananas_author.id
FROM bananas_author
JOIN auth_user ON auth_user.id = bananas_author.user_id
WHERE username = '******'
""" % (_mysql.escape_string(author)))
data = db_conn.store_result().fetch_row(maxrows=0)
if len(data) == 0:
raise MusaException(
"author %s not in bananas; this author must first accept bananas' terms"
% author)
if len(data) > 1:
raise MusaException("duplicate %s authors in bananas" % author)
metadata['resolved_authors'].append(int(data[0][0]))