def ParseAndAddFile(filename, accountid, cfg): from lib import log, upqdb, extract_metadata db = upqdb.UpqDB(cfg.db['url'], cfg.db['debug']) output = SetupLogger() extract_metadata.Extract_metadata(cfg, db, filename, accountid) return output.getvalue()
def __init__(self, db):
dled = {}
self.db = db
#print self.getlobbyversion()
f = download.DownloadFile(stablever, os.path.basename(stablever))
data = json.loads(str(f.read()))
res = upqdb.UpqDB().query(
"SELECT mid from mirror WHERE url_prefix='%s'" % self.prefix)
mid = res.first()[0]
for row in data:
self.update(row, mid)
#delete files that wheren't updated this run (means removed from mirror)
upqdb.UpqDB().query(
"DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s"
% (mid))
return True
def update(self, data, mid):
"""
data is an array with
md5
filectime
version
branch
filesize
os
path
"""
if data['version'] == "testing":
return
filename = self.escape(data['path'][data['path'].rfind("/") + 1:])
category = "engine_" + data['os']
branch = data['branch']
version = data['version']
if not data['branch'] in ('master'):
version = data['version'] + ' ' + data['branch']
url = self.prefix + '/' + data['path']
cid = upqdb.getCID(self.db, category)
#print "%s %s %s %s" % (filename, version, category, url)
try:
fid = upqdb.UpqDB().insert(
"file",
{
"filename": filename,
"name": "spring",
"version": version,
"cid": cid,
"md5": data['md5'],
"timestamp": datetime.datetime.fromtimestamp(
data['filectime']),
#"timestamp": data['filectime'],
"size": data['filesize'],
"status": 1
})
except upqdb.UpqDBIntegrityError as e:
try:
res = upqdb.UpqDB().query(
"SELECT fid from file WHERE version='%s' and cid=%s" %
(version, cid))
fid = res.first()[0]
upqdb.UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s" %
(data['md5'], fid))
except Exception as e:
logging.error("Error %s %s %s", version, cid, e)
return
relpath = self.escape(url[len(self.prefix) + 1:])
try:
id = upqdb.UpqDB().insert(
"mirror_file", {
"mid": mid,
"path": relpath,
"status": 1,
"fid": fid,
"lastcheck": upqdb.UpqDB().now()
})
except upqdb.UpqDBIntegrityError:
res = upqdb.UpqDB().query(
"SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s" %
(mid, fid))
id = res.first()[0]
upqdb.UpqDB().query(
"UPDATE mirror_file SET lastcheck=NOW() WHERE mfid = %s" %
(id))
#!/usr/bin/env python3 # This file is part of the "upq" program used on springfiles.springrts.com to manage file # uploads, mirror distribution etc. It is published under the GPLv3. # #Copyright (C) 2011 Matthias Ableitner (spring #at# abma #dot# de) # #You should have received a copy of the GNU General Public License #along with this program. If not, see <http://www.gnu.org/licenses/>. # sf-sync: syncs file data with springfiles # can be either initiaded by an updated file # or maybe by the xml-rpc interface (or cron?) from lib import log, upqconfig, upqdb, versionfetch import sys import json import os cfg = upqconfig.UpqConfig() db = upqdb.UpqDB(cfg.db['url'], cfg.db['debug']) versionfetch.Versionfetch(cfg, db)
def GetResult(request):
cfg = upqconfig.UpqConfig()
db = upqdb.UpqDB(cfg.db['url'], cfg.db['debug'])
wherecond = ""
if "logical" in request and request["logical"] == "or":
logical = " OR "
else:
logical = " AND "
if "nosensitive" in request:
binary = ""
else:
binary = "BINARY"
conditions = {
"tag": "t.tag LIKE {binary} '{tag}'",
"filename": "f.filename LIKE {binary} '{filename}'",
"category": "c.name LIKE {binary} '{category}'",
"name": "f.name LIKE {binary} '{name}'",
"version": "f.version LIKE {binary}'{version}'",
"sdp": "f.sdp LIKE {binary} '{sdp}'",
# FIXME: concat is really slow!
"springname":
"((f.name LIKE {binary} '{springname}' OR f.version LIKE {binary} '{springname}') OR (CONCAT(f.name,' ',f.version) LIKE {binary} '{springname}'))",
"md5": "f.md5 = '{md5}'",
}
wheres = []
for tag, condition in conditions.items():
wheres += GetQuery(request, binary, tag, condition)
wherecond = logical.join(wheres)
if wherecond:
wherecond = " AND " + wherecond
#print(wherecond)
rows = db.query("""SELECT
distinct(f.fid) as fid,
f.name as name,
f.filename as filename,
f.path as path,
f.md5 as md5,
f.sdp as sdp,
f.version as version,
LOWER(c.name) as category,
f.size as size,
f.timestamp as timestamp,
f.metadata as metadata
FROM file as f
LEFT JOIN categories as c ON f.cid=c.cid
LEFT JOIN tag as t ON f.fid=t.fid
WHERE c.cid>0
AND f.status=1
%s
ORDER BY f.timestamp DESC
%s
""" % (wherecond, getlimit(request)))
clientres = []
for row in rows:
d = dict(row)
#inject local file as mirror
if d["category"] in ["game", "map"]:
d["mirrors"] = [
"https://springfiles.springrts.com/files/" + d["path"] + "/" +
d["filename"]
]
else:
d["mirrors"] = []
d["mirrors"] += GetMirrors(db, d["fid"])
#print(mirrors)
#print(row)
try:
d["metadata"] = json.loads(d["metadata"]) if d["metadata"] else {}
except json.decoder.JSONDecodeError:
d["metadata"] = ""
if "images" in request:
for k in ["splash", "mapimages"]:
if k in d["metadata"]:
d[k] = GetMetadataPaths(d["metadata"][k])
del (d["metadata"][k])
if not "metadata" in request:
del (d["metadata"])
#if "splash" in request:
#if "images" in request:
#json.dumps(row)
d["tags"] = GetTags(db, d["fid"])
del (d["fid"])
if d["timestamp"]:
d["timestamp"] = d["timestamp"].isoformat()
if d["version"] == "":
d["springname"] = d["name"]
else:
d["springname"] = d["name"] + " " + d["version"]
clientres.append(d)
return clientres
def AddFilesToDB(files):
from lib import log, upqdb, extract_metadata
db = upqdb.UpqDB(cfg.db['url'], cfg.db['debug'])
for f in files:
extract_metadata.Extract_metadata(cfg, db, f, 1)
