def getmetadata(self, fid):
"""
returns the metadata, that is required for springfiles in a struct
required data:
link to images
mirror urls
file size
md5
"""
data = {}
data['mirror'] = []
results = UpqDB().query(
"SELECT CONCAT(m.url_prefix, f.path) as url FROM mirror_file f LEFT JOIN mirror m ON f.mid=m.mid WHERE f.fid=%d"
% (fid))
for res in results:
data['mirror'].append(res['url'])
results = UpqDB().query(
"SELECT f.filename, f.size, f.timestamp, f.md5, f.name, f.version, c.name as category, f.metadata FROM file f LEFT JOIN categories c ON f.cid=c.cid WHERE f.fid=%d"
% (fid))
res = results.first()
for value in [
"filename", "size", "timestamp", "md5", "name", "version",
"category"
]:
data[value] = res[value]
if res['metadata'] != None and len(res['metadata']) > 0:
data['metadata'] = json.loads(res['metadata'])
else:
data['metadata'] = []
return data
def run(self):
"""
class Hash must be initialized with fileid!
"""
fid = int(self.jobdata['fid'])
results = UpqDB().query("SELECT filename, path, md5, sha1, sha256 FROM file WHERE fid=%d " % int(fid))
res=results.first()
filename = os.path.join(UpqConfig().paths['files'], res['path'], res['filename'])
if not os.path.exists(filename):
self.msg("File %s doesn't exist" % (filename))
return False
hashes = self.hash(filename)
if res['md5']!=None and res['md5']!=hashes['md5']:
self.logger.error("md5 missmatch")
return False
if res['sha1']!=None and res['sha1']!=hashes['sha1']:
self.logger.error("sha1 missmatch")
return False
if res['sha256']!=None and res['sha256']!=hashes['sha256']:
self.logger.error("sha256 missmatch")
return False
UpqDB().query("UPDATE file set md5='%s', sha1='%s', sha256='%s' WHERE fid=%d" %
(hashes['md5'], hashes['sha1'], hashes['sha256'], fid))
self.msg("md5: %s sha1: %s sha256: %s" % (hashes['md5'], hashes['sha1'], hashes['sha256']))
return True
def normalizeFilename(self, srcfile, fid, name, version):
""" normalize filename + renames file + updates filename in database """
name=name.lower()
if len(version)>0:
name = str(name[:200] +"-" + version.lower())[:255]
_, extension = os.path.splitext(srcfile)
name += extension
res=""
for c in name:
if c in "abcdefghijklmnopqrstuvwxyz-_.01234567890":
res+=c
else:
res+="_"
dstfile=os.path.join(os.path.dirname(srcfile), res)
if srcfile!=dstfile:
results=UpqDB().query("SELECT fid FROM file WHERE BINARY filename='%s' AND status=1" % (res))
row=results.first()
if row or os.path.exists(dstfile):
self.logger.error("Error renaming file: %s to %s already exists, deleting source + using dst!" % (srcfile, dstfile))
os.remove(srcfile)
else:
shutil.move(srcfile, dstfile)
UpqDB().query("UPDATE file SET filename='%s' WHERE fid=%s" %(res, fid))
self.logger.info("Normalized filename to %s "%(dstfile))
return dstfile
def check(self):
if 'file' in self.jobdata:
if not os.path.exists(self.jobdata['file']):
self.logger.error("File %s doesn't exist" %
self.jobdata['file'])
return False
elif 'fid' in self.jobdata:
results = UpqDB().query(
"SELECT filename, path, status FROM file WHERE fid=%s" %
(int(self.jobdata['fid'])))
res = results.first()
if not res:
self.logger.error("Fid not found in db")
return False
prefix = self.getPathByStatus(res['status'])
self.jobdata['file'] = os.path.join(prefix, res['path'],
res['filename'])
if not os.path.exists(self.jobdata['file']):
self.logger.error("filepath from db doesn't exist %s" %
(self.jobdata['file']))
return False
else:
self.logger.error("Either fid or file has to be set")
return False
self.enqueue_job()
return True
def run(self):
"""
class Hash must be initialized with fileid!
"""
fid = int(self.jobdata['fid'])
results = UpqDB().query(
"SELECT filename, path, md5, sha1, sha256 FROM file WHERE fid=%d "
% int(fid))
res = results.first()
filename = os.path.join(UpqConfig().paths['files'], res['path'],
res['filename'])
if not os.path.exists(filename):
self.msg("File %s doesn't exist" % (filename))
return False
hashes = self.hash(filename)
if res['md5'] != None and res['md5'] != hashes['md5']:
self.logger.error("md5 missmatch")
return False
if res['sha1'] != None and res['sha1'] != hashes['sha1']:
self.logger.error("sha1 missmatch")
return False
if res['sha256'] != None and res['sha256'] != hashes['sha256']:
self.logger.error("sha256 missmatch")
return False
UpqDB().query(
"UPDATE file set md5='%s', sha1='%s', sha256='%s' WHERE fid=%d" %
(hashes['md5'], hashes['sha1'], hashes['sha256'], fid))
self.msg("md5: %s sha1: %s sha256: %s" %
(hashes['md5'], hashes['sha1'], hashes['sha256']))
return True
def getCid(self, name):
result=UpqDB().query("SELECT cid from categories WHERE name='%s'" % name)
res=result.first()
if res:
cid=res['cid']
else:
cid=UpqDB().insert("categories", {"name": name})
return cid
def getCid(self, name):
result = UpqDB().query("SELECT cid from categories WHERE name='%s'" %
name)
res = result.first()
if res:
cid = res['cid']
else:
cid = UpqDB().insert("categories", {"name": name})
return cid
def getCID(self, category):
if category in self.cats:
return self.cats[category]
res = UpqDB().query("SELECT cid from categories WHERE name='%s'" % (category))
try:
self.cats[category]=res.first()[0] # cache result
except:
self.logger.error("Invalid category: %s" % category)
return self.cats[category]
def check(self):
results = UpqDB().query("SELECT fid, filename, path FROM file WHERE fid=%d " % int(self.jobdata['fid']))
res=results.first()
filename = os.path.join(UpqConfig().paths['files'], res['path'], res['filename'])
if not os.path.exists(filename):
self.logger.error("file %d doesn't exist: %s"%(res['fid'], filename))
return False
self.enqueue_job()
return True
def insertData(self, data, filename):
metadata=data.copy()
del metadata['Depends'] #remove redundant entries
del metadata['sdp']
del metadata['Version']
del metadata['Name']
metadata=json.dumps(metadata)
if 'fid' in self.jobdata: # detect already existing files
fid=self.jobdata['fid']
else:
results=UpqDB().query("SELECT fid FROM file WHERE sdp='%s'"% (data['sdp']))
res=results.first()
if res:
fid=res['fid']
else:
fid=0
if fid<=0:
fid=UpqDB().insert("file", {
"name": self.escape(data['Name']),
"version": self.escape(data['Version']),
"sdp": data['sdp'],
"cid": self.getCid(data['Type']),
"metadata": metadata,
"uid": 0,
"path": "",
"filename": filename,
"timestamp": UpqDB().now(), #fixme: use file timestamp
"size": os.path.getsize(filename),
"status": 1,
})
else:
UpqDB().query("UPDATE file SET name='%s', version='%s', sdp='%s', cid=%s, metadata='%s' WHERE fid=%s" %(
self.escape(data['Name']),
data['Version'],
data['sdp'],
self.getCid(data['Type']),
metadata,
fid
))
# remove already existing depends
UpqDB().query("DELETE FROM file_depends WHERE fid = %s" % (fid) )
for depend in data['Depends']:
res=UpqDB().query("SELECT fid FROM file WHERE CONCAT(name,' ',version)='%s'" % (depend))
row=res.first()
if not row:
id=0
else:
id=row['fid']
try:
UpqDB().insert("file_depends", {"fid":fid, "depends_string": depend, "depends_fid": id})
self.msg("Added %s '%s' version '%s' to the mirror-system" % (data['Type'], data['Name'], data['Version']))
except UpqDBIntegrityError:
pass
self.msg("Updated %s '%s' version '%s' sdp '%s' in the mirror-system" % (data['Type'], data['Name'], data['Version'], data['sdp']))
return fid
def getCID(self, category):
if category in self.cats:
return self.cats[category]
res = UpqDB().query("SELECT cid from categories WHERE name='%s'" %
(category))
try:
self.cats[category] = res.first()[0] # cache result
except:
self.logger.error("Invalid category: %s" % category)
return self.cats[category]
def check(self):
if not "fid" in self.jobdata:
self.msg("fid not specified")
return False
results = UpqDB().query("SELECT * FROM file WHERE fid=%s and status=1" % (int(self.jobdata["fid"])))
res = results.first()
if res == None:
self.msg("fid not found")
return False
id = self.enqueue_job()
return True
def check(self):
if not 'fid' in self.jobdata:
self.msg("fid not specified")
return False
results=UpqDB().query("SELECT * FROM file WHERE fid=%s and status=1" % (int(self.jobdata['fid'])))
res=results.first()
if res == None:
self.msg("fid not found")
return False
id=self.enqueue_job()
return True
def task_done(self, job):
# update job state in DB
if job.result:
result = 0
else:
result = 1
if len(job.msgstr) > 255: #limit string length
job.msgstr = job.msgstr[:255]
query = "UPDATE upqueue SET status=0, end_time=NOW(), result_msg='%s' WHERE jobid = %s" % (
UpqDB().escape(job.msgstr), job.jobid)
UpqDB().query(query)
super(DBQueue, self).task_done()
def check(self):
results = UpqDB().query(
"SELECT fid, filename, path FROM file WHERE fid=%d " %
int(self.jobdata['fid']))
res = results.first()
filename = os.path.join(UpqConfig().paths['files'], res['path'],
res['filename'])
if not os.path.exists(filename):
self.logger.error("file %d doesn't exist: %s" %
(res['fid'], filename))
return False
self.enqueue_job()
return True
def run(self):
results = UpqDB().query("SELECT count(mid) from mirror")
count = res.first()[0]
results = UpqDB().query(
"SELECT f.mfid FROM mirror_file f LEFT JOIN mirror m ON f.mid=m.mid \
WHERE m.status=1 \
AND ( f.lastcheck < DATE_SUB(NOW(), INTERVAL 1 MONTH) \
OR f.lastcheck is null ) \
ORDER BY f.lastcheck ASC \
LIMIT 0,30")
for res in results:
self.enqueue_newjob("verify_remote_file", {"mfid": res["mfid"]})
sleep(10)
return True
def put(self, job, block=True, timeout=None):
if job.jobid == -1:
# add job to DB in state "new"
jobdata = json.dumps(job.jobdata, -1)
ret = UpqDB().insert(
"upqueue", {
'jobname': job.jobname,
'status': 2,
'jobdata': jobdata,
'ctime': UpqDB().now()
})
job.jobid = ret #set jobid
super(DBQueue, self).put(job, block, timeout)
return job.jobid
def run(self):
dled = {}
url = self.prefix + '/list.php'
#print self.getlobbyversion()
f = my_download().open(url)
data = json.loads(str(f.read()))
res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" % self.prefix)
mid = res.first()[0]
for row in data:
self.update(row, mid)
#delete files that wheren't updated this run (means removed from mirror)
UpqDB().query("DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s" %(mid))
urllib.urlcleanup()
return True
def update(self, data, mid):
"""
data is an array with
md5
filectime
version
branch
filesize
os
path
"""
if data['version'] == "testing":
return
filename = self.escape(data['path'][data['path'].rfind("/") + 1:])
category = "engine_" + data['os']
branch = data['branch']
version = data['version']
if not data['branch'] in ('master'):
version = data['version'] + ' ' + data['branch']
url = self.prefix + '/' + data['path']
cid = self.getCID(category)
#print "%s %s %s %s" % (filename, version, category, url)
try:
fid = UpqDB().insert(
"file",
{
"filename": filename,
"name": "spring",
"version": version,
"cid": cid,
"md5": data['md5'],
"timestamp": datetime.datetime.fromtimestamp(
data['filectime']),
#"timestamp": data['filectime'],
"size": data['filesize'],
"status": 1
})
except UpqDBIntegrityError, e:
try:
res = UpqDB().query(
"SELECT fid from file WHERE version='%s' and cid=%s" %
(version, cid))
fid = res.first()[0]
UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s" %
(data['md5'], fid))
except Exception, e:
self.logger.error("Error %s %s %s", version, cid, e)
return
def run(self):
fid=int(self.jobdata['fid'])
results=UpqDB().query("SELECT filename, path, sdp FROM file WHERE fid=%d AND status=1" % fid)
res=results.first()
#filename of the archive to be scanned
filename=res['filename'] # filename only (no path info)
absfilename=os.path.join(UpqConfig().paths['files'], res['path'], res['filename']) # absolute filename
torrent=os.path.join(UpqConfig().paths['metadata'], res['sdp']+ ".torrent" )
if not os.path.exists(absfilename):
self.msg("File doesn't exist: %s" %(absfilename))
return False
res=self.create_torrent(absfilename, torrent)
if res:
UpqDB().query("UPDATE file SET torrent=1 WHERE fid=%s" %(fid))
return res
def run(self):
fid = int(self.jobdata["fid"])
results = UpqDB().query("SELECT filename, path, sdp FROM file WHERE fid=%d AND status=1" % fid)
res = results.first()
# filename of the archive to be scanned
filename = res["filename"] # filename only (no path info)
absfilename = os.path.join(UpqConfig().paths["files"], res["path"], res["filename"]) # absolute filename
torrent = os.path.join(UpqConfig().paths["metadata"], res["sdp"] + ".torrent")
if not os.path.exists(absfilename):
self.msg("File doesn't exist: %s" % (absfilename))
return False
res = self.create_torrent(absfilename, torrent)
if res:
UpqDB().query("UPDATE file SET torrent=1 WHERE fid=%s" % (fid))
return res
def check(self):
if 'file' in self.jobdata :
self.jobdata['filename']=os.path.basename(self.jobdata['file'])
result=UpqDB().query("SELECT * from file where filename LIKE '%s'" % (self.jobdata['filename']))
res=result.first()
if res!=None:
self.jobdata['fid']=res['fid']
self.msg("File %s already exists: fid: %s"%(self.jobdata['file'], res['fid']))
if not os.access(self.jobdata['file'], os.R_OK):
self.msg("Can't access %s"% self.jobdata['file'])
return False
elif not 'fid' in self.jobdata:
self.msg("Either file or fid has to be set!")
return False
self.enqueue_job()
return True
def get(self, block=True, timeout=None):
job = super(DBQueue, self).get(block, timeout)
# update job state in DB
query = "UPDATE upqueue SET status=1, start_time=NOW() WHERE jobid = %d" % (
job.jobid)
UpqDB().query(query)
return job
def run(self):
dled = {}
url = self.prefix + '/list.php'
#print self.getlobbyversion()
f = my_download().open(url)
data = json.loads(str(f.read()))
res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" %
self.prefix)
mid = res.first()[0]
for row in data:
self.update(row, mid)
#delete files that wheren't updated this run (means removed from mirror)
UpqDB().query(
"DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s"
% (mid))
urllib.urlcleanup()
return True
def run(self):
"""
params:
filepath: absoulte path of file
filename: filename
uid: uid
or:
fid
"""
if 'fid' in self.jobdata: # file already exists in db, set filename
result=UpqDB().query("SELECT * from file where fid=%s"% (self.jobdata['fid']))
res=result.first()
if res==None:
self.msg("fid not found in db!")
return False
self.msg("File already known, Filename: %s Size: %d" % (res['filename'], res['size']))
return True
def UpdateSDP(self, sdp):
"""
values = {
"tag"=sdp[0],
"md5"=sdp[1],
"depends"=sdp[2],
"name"=sdp[3],
}
"""
#check if file is already known
res=UpqDB().query("SELECT f.fid FROM file f \
LEFT JOIN tag t ON t.fid=f.fid \
WHERE sdp='%s'" % (sdp[1]))
row=res.first()
if row: #file is already known
#delete tag from existing files
UpqDB().query("DELETE FROM tag WHERE tag='%s'" % (sdp[0]))
#insert updated tag
UpqDB().query("INSERT INTO tag (fid, tag) VALUES (%s, '%s')" % (row['fid'], sdp[0]))
#self.logger.info("updated %s %s %s %s",sdp[3],sdp[2],sdp[1],sdp[0])
return
if not sdp[3]: #without a name, we can't do anything!
return
cid = self.getCid("game")
try:
fid = UpqDB().insert("file", {
"filename" : sdp[3] + " (not available as sdz)",
"name": sdp[3],
"cid" : cid,
"md5" : sdp[1],
"sdp" : sdp[1],
"size" : 0,
"status" : 4, # special status for files that can only be downloaded via rapid
"uid" : 0,
"path" : "",
})
UpqDB().query("INSERT INTO tag (fid, tag) VALUES (%s, '%s')" % (fid, sdp[0]))
#self.logger.info("inserted %s %s %s %s",sdp[3],sdp[2],sdp[1],sdp[0])
except Exception as e:
self.logger.error(str(e))
self.logger.error("Error from sdp: %s %s %s %s", sdp[3], sdp[2],sdp[1],sdp[0])
res=UpqDB().query("SELECT * FROM file f WHERE name='%s'" % sdp[3])
if res:
row=res.first()
self.logger.error("a file with this name already exists, fid=%s, sdp=%s" % (row['fid'], row['sdp']))
def ExtractMetadata(self, usync, archiveh, filename, filepath,
metadatapath, hashes):
filelist = self.getFileList(usync, archiveh)
sdp = self.getSDPName(usync, archiveh)
idx = self.getMapIdx(usync, filename)
if idx >= 0: #file is map
archivepath = usync.GetArchivePath(
filename.encode()).decode() + filename
springname = usync.GetMapName(idx).decode()
data = self.getMapData(usync, filename, idx, archiveh, springname)
data['mapimages'] = self.dumpmap(usync, springname, metadatapath,
filename, idx)
data['path'] = self.jobcfg['maps-path']
else: # file is a game
idx = self.getGameIdx(usync, filename)
if idx < 0:
self.logger.error("Invalid file detected: %s %s %s" %
(filename, usync.GetNextError(), idx))
return False
self.logger.debug("Extracting data from " + filename)
archivepath = usync.GetArchivePath(filename) + filename
gamearchivecount = usync.GetPrimaryModArchiveCount(
idx) # initialization for GetPrimaryModArchiveList()
data = self.getGameData(usync, idx, gamearchivecount, archivepath,
archiveh)
data['path'] = self.jobcfg['games-path']
if (sdp == "") or (data['Name'] == ""
): #mark as broken because sdp / name is missing
self.logger.error("Couldn't get name / filename")
return False
data['splash'] = self.createSplashImages(usync, archiveh, filelist)
_, extension = os.path.splitext(filename)
moveto = os.path.join(
self.getPathByStatus(1), data['path'],
self.GetNormalizedFilename(data['Name'], data['Version'],
extension))
self.jobdata['file'] = moveto
assert (moveto != filepath)
assert (len(moveto) > 0)
if not self.movefile(filepath, moveto):
self.logger.error("Couldn't move file %s -> %s" %
(filepath, moveto))
return False
try:
data['sdp'] = sdp
self.jobdata['fid'] = self.insertData(data, moveto, hashes)
except UpqDBIntegrityError:
self.logger.error("Duplicate file detected: %s %s %s" %
(filename, data['Name'], data['Version']))
return False
if self.create_torrent(filepath,
os.path.join(metadatapath, sdp, '.torrent')):
UpqDB().query("UPDATE file SET torrent=1 WHERE fid=%s" % (fid))
return True
def check(self):
if 'file' in self.jobdata:
self.jobdata['filename'] = os.path.basename(self.jobdata['file'])
result = UpqDB().query(
"SELECT * from file where filename LIKE '%s'" %
(self.jobdata['filename']))
res = result.first()
if res != None:
self.jobdata['fid'] = res['fid']
self.msg("File %s already exists: fid: %s" %
(self.jobdata['file'], res['fid']))
if not os.access(self.jobdata['file'], os.R_OK):
self.msg("Can't access %s" % self.jobdata['file'])
return False
elif not 'fid' in self.jobdata:
self.msg("Either file or fid has to be set!")
return False
self.enqueue_job()
return True
def run(self):
"""
params:
filepath: absoulte path of file
filename: filename
uid: uid
or:
fid
"""
if 'fid' in self.jobdata: # file already exists in db, set filename
result = UpqDB().query("SELECT * from file where fid=%s" %
(self.jobdata['fid']))
res = result.first()
if res == None:
self.msg("fid not found in db!")
return False
self.msg("File already known, Filename: %s Size: %d" %
(res['filename'], res['size']))
return True
def update(self, data, mid):
"""
data is an array with
md5
filectime
version
branch
filesize
os
path
"""
if data['version'] == "testing":
return
filename = self.escape(data['path'][data['path'].rfind("/")+1:])
category = "engine_" + data['os']
branch = data['branch']
version = data['version']
if not data['branch'] in ('master'):
version = data['version'] + ' ' + data['branch']
url = self.prefix +'/' + data['path']
cid = self.getCID(category)
#print "%s %s %s %s" % (filename, version, category, url)
try:
fid = UpqDB().insert("file", {
"filename" : filename,
"name": "spring",
"version": version,
"cid" : cid,
"md5" : data['md5'],
"timestamp": datetime.datetime.fromtimestamp(data['filectime']),
#"timestamp": data['filectime'],
"size": data['filesize'],
"status": 1 })
except UpqDBIntegrityError, e:
try:
res = UpqDB().query("SELECT fid from file WHERE version='%s' and cid=%s" % (version, cid))
fid = res.first()[0]
UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s"% (data['md5'], fid))
except Exception, e:
self.logger.error("Error %s %s %s", version, cid, e)
return
def FixPathes(self):
from jobs import extract_metadata
rows = UpqDB().query(
"select filename, fid, name, version, cid, status from file where filename like '/tmp%%'"
)
j = extract_metadata.Extract_metadata("extract_metadata", {})
for row in rows:
#print(row)
srcfile = j.normalizeFilename(row[0], row[2], row[3])
subdir = j.jobcfg['maps-path'] if row[4] == 1 else j.jobcfg[
'games-path']
status = row[5]
#self.movefile(filepath, 1, moveto)
prefix = j.getPathByStatus(status)
dstfile = os.path.join(prefix, subdir, os.path.basename(srcfile))
filename = os.path.basename(srcfile)
UpqDB().query(
"UPDATE file SET path='%s', status=%d, filename='%s' WHERE fid=%d"
% (subdir, status, filename, row[1]))
def check(self):
if 'file' in self.jobdata:
if not os.path.exists(self.jobdata['file']):
self.logger.error("File %s doesn't exist" % self.jobdata['file'])
return False
elif 'fid' in self.jobdata:
results=UpqDB().query("SELECT filename, path, status FROM file WHERE fid=%s" % (int(self.jobdata['fid'])))
res=results.first()
if not res:
self.logger.error("Fid not found in db")
return False
prefix=self.getPathByStatus(res['status'])
self.jobdata['file']=os.path.join(prefix, res['path'], res['filename'] )
if not os.path.exists(self.jobdata['file']):
self.logger.error("filepath from db doesn't exist %s" %(self.jobdata['file']))
return False
else:
self.logger.error("Either fid or file has to be set")
return False
self.enqueue_job()
return True
def run(self):
#username=self.jobcfg['username']
#password=self.jobcfg['password']
proxy = ServerProxy("https://springfiles.com/xmlrpc.php")
row = UpqDB().query("SELECT MAX(sid) FROM sf_sync2").first()
if row:
lastsid = int(row[0]) if row[0] else 0
else:
lastsid = 0
self.logger.info("Syncing %d" % (lastsid))
files = proxy.springfiles.getfiles(lastsid)
for data in files:
sid = int(data["fid"])
row = UpqDB().query("SELECT sid FROM sf_sync2 WHERE sid=%d" %
(int(sid))).first()
if row: # file found, nothing to do
self.logger.debug("File mapping found, continue")
continue
self.logger.debug("%s %s" % (sid, data))
row = UpqDB().query("SELECT fid FROM file WHERE md5='%s'" %
(data["md5"])).first()
if row:
#UpqDB().insert("sf_sync", {"sid": sid, "fid": row["fid"]})
UpqDB().query(
"INSERT INTO sf_sync2 (sid, fid) VALUES (%d, %d) " %
(sid, row["fid"]))
self.logger.info("Added mapping: %s <-> %s" %
(sid, row["fid"]))
continue
if not self.download_and_add_file("https://springfiles.com/" +
data["filepath"]):
self.logger.error("Error adding %s" % (data))
else:
row = UpqDB().query("SELECT fid FROM file WHERE md5='%s'" %
(data["md5"])).first()
assert (row)
return True
def getmetadata(self, fid):
"""
returns the metadata, that is required for springfiles in a struct
required data:
link to images
mirror urls
file size
md5
"""
data={}
data['mirror']=[]
results=UpqDB().query("SELECT CONCAT(m.url_prefix, f.path) as url FROM mirror_file f LEFT JOIN mirror m ON f.mid=m.mid WHERE f.fid=%d" % (fid))
for res in results:
data['mirror'].append(res['url'])
results=UpqDB().query("SELECT f.filename, f.size, f.timestamp, f.md5, f.name, f.version, c.name as category, f.metadata FROM file f LEFT JOIN categories c ON f.cid=c.cid WHERE f.fid=%d" %(fid))
res=results.first()
for value in ["filename", "size", "timestamp", "md5", "name", "version", "category"]:
data[value]=res[value]
if res['metadata']!=None and len(res['metadata'])>0:
data['metadata']=json.loads(res['metadata'])
else:
data['metadata']=[]
return data
def run(self):
dled = {}
url = self.prefix + '/list.php'
filename = "/tmp/sprinvers.json"
if not self.DownloadFile(url, filename):
self.logger.info("list.php wasn't changed")
return True
with open(filename, "r") as f:
data = json.loads(str(f.read()))
res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" %
self.prefix)
mid = res.first()[0]
assert (mid > 0)
for row in data:
self.update(row, mid)
#delete files that wheren't updated this run (means removed from mirror)
UpqDB().query(
"DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s"
% (mid))
return True
def rpc_call_sync(self, proxy, username, password, data):
"""
make the xml-rpc call
"""
try:
rpcres = proxy.springfiles.sync(username, password, data)
for rpc in rpcres: #set description url received from springfiles
UpqDB().query(
"UPDATE file SET descriptionuri='%s' WHERE fid=%d" %
(rpc['url'], rpc['fid']))
except Exception as e:
self.msg("xmlrpc springfiles.sync() error: %s" % (e))
return True # fixme
self.logger.debug("received from springfiles: %s", rpcres)
return True
def deleteOldFiles(self):
"""
deletes files from ftpmirror when:
file is older than 100 days
it is known in rapid
it is not a stable|test tag (no number at the end in tag)
"""
results = UpqDB().query(
"SELECT m.mid, ftp_url, ftp_user, ftp_pass, ftp_dir, ftp_port, ftp_passive, ftp_ssl \
FROM mirror as m \
WHERE m.status=1")
for mirror in results:
results2 = UpqDB().query("SELECT f.fid,tag FROM `file` f \
LEFT JOIN tag t ON f.fid=t.fid \
WHERE (t.fid>0) \
AND f.status=1 \
AND timestamp < NOW() - INTERVAL 100 DAY \
GROUP BY f.fid HAVING count(f.fid) = 1")
ftp = None
for filetodel in results2:
if not ftp:
try:
ftp = self.ftpconnect(
mirror['ftp_url'], mirror['ftp_port'],
mirror['ftp_user'], mirror['ftp_pass'],
mirror['ftp_passive'], mirror['ftp_dir'],
mirror['ftp_ssl'])
except Exception as e:
self.logger.error("Couldn't connect to %s: %s",
mirror['ftp_url'], e)
break
res2 = UpqDB().query(
"SELECT * FROM mirror_file WHERE fid = %d AND mid = %d AND status=1"
% (filetodel['fid'], mirror['mid']))
curfile = res2.first()
if curfile and filetodel['tag'][-1].isdigit():
try:
self.logger.error("deleting %s" % (curfile['path']))
ftp.delete(curfile['path'])
except:
pass
UpqDB().query(
"UPDATE mirror_file SET status=4 WHERE mfid = %d" %
(int(curfile['mfid'])))
if ftp:
ftp.close()
try:
res = UpqDB().query(
"SELECT fid from file WHERE version='%s' and cid=%s" %
(version, cid))
fid = res.first()[0]
UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s" %
(data['md5'], fid))
except Exception, e:
self.logger.error("Error %s %s %s", version, cid, e)
return
relpath = self.escape(url[len(self.prefix) + 1:])
try:
id = UpqDB().insert(
"mirror_file", {
"mid": mid,
"path": relpath,
"status": 1,
"fid": fid,
"lastcheck": UpqDB().now()
})
except UpqDBIntegrityError:
res = UpqDB().query(
"SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s" %
(mid, fid))
id = res.first()[0]
UpqDB().query(
"UPDATE mirror_file SET lastcheck=NOW() WHERE mfid = %s" %
(id))
def run(self):
dled = {}
url = self.prefix + '/list.php'
def insertData(self, data, filename, hashes):
metadata = data.copy()
del metadata['Depends'] #remove redundant entries
del metadata['sdp']
del metadata['Version']
del metadata['Name']
self.logger.debug(metadata)
metadata = json.dumps(metadata)
if 'fid' in self.jobdata: # detect already existing files
fid = self.jobdata['fid']
else:
results = UpqDB().query(
"SELECT fid FROM file WHERE sdp='%s' or md5='%s'" %
(data['sdp'], hashes['md5']))
res = results.first()
if res:
fid = res['fid']
else:
fid = 0
if fid <= 0:
fid = UpqDB().insert(
"file",
{
"name": self.escape(data['Name']),
"version": self.escape(data['Version']),
"sdp": data['sdp'],
"cid": self.getCid(data['Type']),
"metadata": metadata,
"uid": 0,
"path": data["path"],
"filename": os.path.basename(filename),
"timestamp": UpqDB().now(), #fixme: use file timestamp
"size": os.path.getsize(filename),
"status": 1,
"md5": hashes["md5"],
"sha1": hashes["sha1"],
"sha256": hashes["sha256"],
})
else:
UpqDB().query(
"UPDATE file SET name='%s', version='%s', sdp='%s', cid=%s, metadata='%s', md5='%s', sha1='%s', sha256='%s', status=1 WHERE fid=%s"
% (self.escape(data['Name']), data['Version'], data['sdp'],
self.getCid(data['Type']), metadata, hashes["md5"],
hashes["sha1"], hashes["sha256"], fid))
# remove already existing depends
UpqDB().query("DELETE FROM file_depends WHERE fid = %s" % (fid))
for depend in data['Depends']:
res = UpqDB().query(
"SELECT fid FROM file WHERE CONCAT(name,' ',version)='%s'" %
(depend))
row = res.first()
if not row:
id = 0
else:
id = row['fid']
try:
UpqDB().insert("file_depends", {
"fid": fid,
"depends_string": depend,
"depends_fid": id
})
self.msg("Added %s '%s' version '%s' to the mirror-system" %
(data['Type'], data['Name'], data['Version']))
except UpqDBIntegrityError:
pass
self.msg("Updated %s '%s' version '%s' sdp '%s' in the mirror-system" %
(data['Type'], data['Name'], data['Version'], data['sdp']))
return fid
def update(self, data, mid):
"""
data is an array with
md5
filectime
version
branch
filesize
os
path
"""
if data['version'] == "testing":
return
filename = self.escape(data['path'][data['path'].rfind("/") + 1:])
category = "engine_" + data['os']
branch = data['branch']
version = data['version']
if not data['branch'] in ('master'):
version = data['version'] + ' ' + data['branch']
url = self.prefix + '/' + data['path']
cid = self.getCID(category)
#print "%s %s %s %s" % (filename, version, category, url)
res = UpqDB().query(
"SELECT fid from file WHERE version='%s' and cid=%s" %
(version, cid))
fid = res.first()
if fid:
fid = fid[0]
assert (fid > 0)
UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s" %
(data['md5'], fid))
else:
fid = UpqDB().insert(
"file",
{
"filename": filename,
"name": "spring",
"version": version,
"cid": cid,
"md5": data['md5'],
"timestamp": datetime.datetime.fromtimestamp(
data['filectime']),
#"timestamp": data['filectime'],
"size": data['filesize'],
"status": 1
})
res = UpqDB().query(
"SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s" %
(mid, fid))
mfid = res.first()
if mfid:
mfid = mfid[0]
assert (mid > 0)
UpqDB().query(
"UPDATE mirror_file SET lastcheck=NOW() WHERE mfid = %s" %
(mfid))
else:
relpath = self.escape(url[len(self.prefix) + 1:])
mfid = UpqDB().insert(
"mirror_file", {
"mid": mid,
"path": relpath,
"status": 1,
"fid": fid,
"lastcheck": UpqDB().now()
})
def run(self):
fid = int(self.jobdata['fid'])
results = UpqDB().query(
"SELECT filename, path, size, md5 FROM file where fid=%d AND status=1"
% (fid))
if results.rowcount != 1:
self.msg("Wrong result count with fid %d" % fid)
return False
res = results.first()
srcfilename = os.path.join(UpqConfig().paths['files'], res['path'],
res['filename'])
dstfilename = os.path.join(res['path'], res['filename'])
filesize = res['size']
md5 = res['md5']
#uploads a fid to all mirrors
results = UpqDB().query(
"SELECT m.mid, ftp_url, ftp_user, ftp_pass, ftp_dir, ftp_port, ftp_passive, ftp_ssl \
from file f \
left join mirror m on m.status=f.status \
where f.fid=%d and f.status=1 and m.status=1 \
and m.mid not in (select mid from mirror_file where fid=%d and status = 1)"
% (fid, fid))
uploadcount = 0
for res in results:
try:
ftp = self.ftpconnect(res['ftp_url'], res['ftp_port'],
res['ftp_user'], res['ftp_pass'],
res['ftp_passive'], res['ftp_dir'],
res['ftp_ssl'])
except Exception as e:
self.logger.error("Couldn't connect to the ftp server: %s", e)
continue
if not os.path.isfile(srcfilename):
self.msg("File doesn't exist: " + srcfilename)
return False
try:
f = open(srcfilename, "rb")
dstdir = os.path.dirname(dstfilename)
try:
self.logger.debug("cd into " + dstdir)
ftp.cwd(dstdir)
except:
try:
self.logger.debug("mkdir " + dstdir)
ftp.mkd(dstdir)
self.logger.debug("cwd " + dstdir)
ftp.cwd(dstdir)
except:
self.logger.error(
"couldn't cd/mkdir %s, skipping upload " %
(dstdir))
continue
self.logger.info("uploading %s to %s" %
(os.path.basename(dstfilename),
(res['ftp_url'])))
ftp.storbinary('STOR ' + os.path.basename(dstfilename), f)
ftp.quit()
f.close()
try: #upload succeed, mark in db as uploaded
id = UpqDB().insert(
"mirror_file", {
"mid": res['mid'],
"fid": fid,
"path": dstfilename,
"status": 1
})
except UpqDBIntegrityError:
res = UpqDB().query(
"SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s"
% (res['mid'], fid))
id = res.first()
self.logger.info("file already uploaded: mfid=%d" % (id))
UpqDB().query(
"UPDATE mirror_file SET status=1 path='%s' WHERE mfid=%s"
% (dstfilename, id))
self.logger.debug("inserted into db as %d", id)
self.enqueue_newjob("verify_remote_file", {"mfid": id})
except ftplib.all_errors as e:
self.logger.error("Ftp-Error (%s) %s failed %s" %
((res['ftp_url'], srcfilename, e)))
except Exception as e:
self.logger.error("Upload (%s) %s failed %s" %
((res['ftp_url'], srcfilename, e)))
return False
uploadcount += 1
self.msg("Uploaded to %d mirrors." % (uploadcount))
self.deleteOldFiles()
return True
fid = res.first()[0]
UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s"% (data['md5'], fid))
except Exception, e:
self.logger.error("Error %s %s %s", version, cid, e)
return
relpath = self.escape(url[len(self.prefix)+1:])
try:
id = UpqDB().insert("mirror_file", {
"mid" : mid,
"path": relpath,
"status": 1,
"fid": fid,
"lastcheck": UpqDB().now()
})
except UpqDBIntegrityError:
res = UpqDB().query("SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s" % (mid, fid))
id = res.first()[0]
UpqDB().query("UPDATE mirror_file SET lastcheck=NOW() WHERE mfid = %s"% (id))
def run(self):
dled = {}
url = self.prefix + '/list.php'
#print self.getlobbyversion()
f = my_download().open(url)
data = json.loads(str(f.read()))
res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" % self.prefix)
mid = res.first()[0]
for row in data:
self.update(row, mid)
#delete files that wheren't updated this run (means removed from mirror)
UpqDB().query("DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s" %(mid))
def updatefile(self, fid, command):
UpqDB().insert("sf_sync", {'fid': fid, 'command': command})
def run(self):
if self.jobdata.has_key('fid'): #fid set, add change to db
fid = int(self.jobdata['fid'])
command = 0 #default to update command
if self.jobdata.has_key('command'):
command = self.jobdata['command']
if command == "update":
command = 0
elif command == "delete":
command = 1
else: #unknown command
self.logger.error(
"Invalid command: %s, only update or delete are valid"
% (command))
return False
self.updatefile(fid, command)
username = self.jobcfg['username']
password = self.jobcfg['password']
proxy = ServerProxy(self.jobcfg['rpcurl'])
if self.jobdata.has_key(
'sid'): #sid set, update to springfiles requested
sid = int(self.jobdata['sid'])
else: #sid not set, request from springfiles
try:
sid = int(proxy.springfiles.getlastsyncid(username, password))
self.logger.debug("Fetched sid from springfiles: %d" % (sid))
except Exception as e:
self.msg("xmlrpc springfiles.getlastsyncid() error: %s" % (e))
return False
if sid > 0:
results = UpqDB().query(
"SELECT fid, command FROM sf_sync WHERE sid>%d ORDER BY sid " %
(sid)) #get all changes from db
else:
results = UpqDB().query(
"SELECT fid FROM file WHERE status=1 ORDER BY fid"
) #full sync, get all fids
self.logger.debug("springfiles returned %d, doing full sync" %
(sid))
lastfid = -1
lastcmd = -1
requests = []
maxsid = UpqDB().query(
"SELECT MAX(sid) as sid FROM sf_sync").first()['sid']
for res in results: #fixme: use yield
data = {}
if sid == 0: #full sync, all commands are update
data['command'] = "update"
data['metadata'] = self.getmetadata(res['fid'])
else:
if lastfid == res['fid'] and lastcmd == res[
'command']: #skip if the same command was already done before
continue
lastfid = res['fid']
if 'command' in data:
lastcmd = data['command']
if res['command'] == 1: # delete
data['command'] = "delete"
elif res['command'] == 0: # update
data['metadata'] = self.getmetadata(res['fid'])
data['command'] = "update"
else:
self.logger.error("unknown command %d for fid %d",
res['command'], res['fid'])
continue
self.logger.debug("adding sid %d to requests" % (res['fid']))
data['fid'] = res['fid']
data['sid'] = maxsid
requests.append(data)
if len(requests) > int(
self.jobcfg['maxrequests']
): #more then maxrequests queued, flush them
self.logger.debug("flushing %d requests" % (len(requests)))
if not self.rpc_call_sync(proxy, username, password, requests):
return False
requests = []
if len(requests) > 0: #update remaining requests
return self.rpc_call_sync(proxy, username, password, requests)
return True