def insertData(self, data, filename):
metadata=data.copy()
del metadata['Depends'] #remove redundant entries
del metadata['sdp']
del metadata['Version']
del metadata['Name']
metadata=json.dumps(metadata)
if 'fid' in self.jobdata: # detect already existing files
fid=self.jobdata['fid']
else:
results=UpqDB().query("SELECT fid FROM file WHERE sdp='%s'"% (data['sdp']))
res=results.first()
if res:
fid=res['fid']
else:
fid=0
if fid<=0:
fid=UpqDB().insert("file", {
"name": self.escape(data['Name']),
"version": self.escape(data['Version']),
"sdp": data['sdp'],
"cid": self.getCid(data['Type']),
"metadata": metadata,
"uid": 0,
"path": "",
"filename": filename,
"timestamp": UpqDB().now(), #fixme: use file timestamp
"size": os.path.getsize(filename),
"status": 1,
})
else:
UpqDB().query("UPDATE file SET name='%s', version='%s', sdp='%s', cid=%s, metadata='%s' WHERE fid=%s" %(
self.escape(data['Name']),
data['Version'],
data['sdp'],
self.getCid(data['Type']),
metadata,
fid
))
# remove already existing depends
UpqDB().query("DELETE FROM file_depends WHERE fid = %s" % (fid) )
for depend in data['Depends']:
res=UpqDB().query("SELECT fid FROM file WHERE CONCAT(name,' ',version)='%s'" % (depend))
row=res.first()
if not row:
id=0
else:
id=row['fid']
try:
UpqDB().insert("file_depends", {"fid":fid, "depends_string": depend, "depends_fid": id})
self.msg("Added %s '%s' version '%s' to the mirror-system" % (data['Type'], data['Name'], data['Version']))
except UpqDBIntegrityError:
pass
self.msg("Updated %s '%s' version '%s' sdp '%s' in the mirror-system" % (data['Type'], data['Name'], data['Version'], data['sdp']))
return fid
def normalizeFilename(self, srcfile, fid, name, version):
""" normalize filename + renames file + updates filename in database """
name=name.lower()
if len(version)>0:
name = str(name[:200] +"-" + version.lower())[:255]
_, extension = os.path.splitext(srcfile)
name += extension
res=""
for c in name:
if c in "abcdefghijklmnopqrstuvwxyz-_.01234567890":
res+=c
else:
res+="_"
dstfile=os.path.join(os.path.dirname(srcfile), res)
if srcfile!=dstfile:
results=UpqDB().query("SELECT fid FROM file WHERE BINARY filename='%s' AND status=1" % (res))
row=results.first()
if row or os.path.exists(dstfile):
self.logger.error("Error renaming file: %s to %s already exists, deleting source + using dst!" % (srcfile, dstfile))
os.remove(srcfile)
else:
shutil.move(srcfile, dstfile)
UpqDB().query("UPDATE file SET filename='%s' WHERE fid=%s" %(res, fid))
self.logger.info("Normalized filename to %s "%(dstfile))
return dstfile
def run(self):
"""
class Hash must be initialized with fileid!
"""
fid = int(self.jobdata['fid'])
results = UpqDB().query(
"SELECT filename, path, md5, sha1, sha256 FROM file WHERE fid=%d "
% int(fid))
res = results.first()
filename = os.path.join(UpqConfig().paths['files'], res['path'],
res['filename'])
if not os.path.exists(filename):
self.msg("File %s doesn't exist" % (filename))
return False
hashes = self.hash(filename)
if res['md5'] != None and res['md5'] != hashes['md5']:
self.logger.error("md5 missmatch")
return False
if res['sha1'] != None and res['sha1'] != hashes['sha1']:
self.logger.error("sha1 missmatch")
return False
if res['sha256'] != None and res['sha256'] != hashes['sha256']:
self.logger.error("sha256 missmatch")
return False
UpqDB().query(
"UPDATE file set md5='%s', sha1='%s', sha256='%s' WHERE fid=%d" %
(hashes['md5'], hashes['sha1'], hashes['sha256'], fid))
self.msg("md5: %s sha1: %s sha256: %s" %
(hashes['md5'], hashes['sha1'], hashes['sha256']))
return True
def run(self):
"""
class Hash must be initialized with fileid!
"""
fid = int(self.jobdata['fid'])
results = UpqDB().query("SELECT filename, path, md5, sha1, sha256 FROM file WHERE fid=%d " % int(fid))
res=results.first()
filename = os.path.join(UpqConfig().paths['files'], res['path'], res['filename'])
if not os.path.exists(filename):
self.msg("File %s doesn't exist" % (filename))
return False
hashes = self.hash(filename)
if res['md5']!=None and res['md5']!=hashes['md5']:
self.logger.error("md5 missmatch")
return False
if res['sha1']!=None and res['sha1']!=hashes['sha1']:
self.logger.error("sha1 missmatch")
return False
if res['sha256']!=None and res['sha256']!=hashes['sha256']:
self.logger.error("sha256 missmatch")
return False
UpqDB().query("UPDATE file set md5='%s', sha1='%s', sha256='%s' WHERE fid=%d" %
(hashes['md5'], hashes['sha1'], hashes['sha256'], fid))
self.msg("md5: %s sha1: %s sha256: %s" % (hashes['md5'], hashes['sha1'], hashes['sha256']))
return True
def getmetadata(self, fid):
"""
returns the metadata, that is required for springfiles in a struct
required data:
link to images
mirror urls
file size
md5
"""
data = {}
data['mirror'] = []
results = UpqDB().query(
"SELECT CONCAT(m.url_prefix, f.path) as url FROM mirror_file f LEFT JOIN mirror m ON f.mid=m.mid WHERE f.fid=%d"
% (fid))
for res in results:
data['mirror'].append(res['url'])
results = UpqDB().query(
"SELECT f.filename, f.size, f.timestamp, f.md5, f.name, f.version, c.name as category, f.metadata FROM file f LEFT JOIN categories c ON f.cid=c.cid WHERE f.fid=%d"
% (fid))
res = results.first()
for value in [
"filename", "size", "timestamp", "md5", "name", "version",
"category"
]:
data[value] = res[value]
if res['metadata'] != None and len(res['metadata']) > 0:
data['metadata'] = json.loads(res['metadata'])
else:
data['metadata'] = []
return data
def check(self):
if 'file' in self.jobdata:
if not os.path.exists(self.jobdata['file']):
self.logger.error("File %s doesn't exist" %
self.jobdata['file'])
return False
elif 'fid' in self.jobdata:
results = UpqDB().query(
"SELECT filename, path, status FROM file WHERE fid=%s" %
(int(self.jobdata['fid'])))
res = results.first()
if not res:
self.logger.error("Fid not found in db")
return False
prefix = self.getPathByStatus(res['status'])
self.jobdata['file'] = os.path.join(prefix, res['path'],
res['filename'])
if not os.path.exists(self.jobdata['file']):
self.logger.error("filepath from db doesn't exist %s" %
(self.jobdata['file']))
return False
else:
self.logger.error("Either fid or file has to be set")
return False
self.enqueue_job()
return True
def getCid(self, name):
result=UpqDB().query("SELECT cid from categories WHERE name='%s'" % name)
res=result.first()
if res:
cid=res['cid']
else:
cid=UpqDB().insert("categories", {"name": name})
return cid
def UpdateSDP(self, sdp):
"""
values = {
"tag"=sdp[0],
"md5"=sdp[1],
"depends"=sdp[2],
"name"=sdp[3],
}
"""
#check if file is already known
res=UpqDB().query("SELECT f.fid FROM file f \
LEFT JOIN tag t ON t.fid=f.fid \
WHERE sdp='%s'" % (sdp[1]))
row=res.first()
if row: #file is already known
#delete tag from existing files
UpqDB().query("DELETE FROM tag WHERE tag='%s'" % (sdp[0]))
#insert updated tag
UpqDB().query("INSERT INTO tag (fid, tag) VALUES (%s, '%s')" % (row['fid'], sdp[0]))
#self.logger.info("updated %s %s %s %s",sdp[3],sdp[2],sdp[1],sdp[0])
return
if not sdp[3]: #without a name, we can't do anything!
return
cid = self.getCid("game")
try:
fid = UpqDB().insert("file", {
"filename" : sdp[3] + " (not available as sdz)",
"name": sdp[3],
"cid" : cid,
"md5" : sdp[1],
"sdp" : sdp[1],
"size" : 0,
"status" : 4, # special status for files that can only be downloaded via rapid
"uid" : 0,
"path" : "",
})
UpqDB().query("INSERT INTO tag (fid, tag) VALUES (%s, '%s')" % (fid, sdp[0]))
#self.logger.info("inserted %s %s %s %s",sdp[3],sdp[2],sdp[1],sdp[0])
except Exception as e:
self.logger.error(str(e))
self.logger.error("Error from sdp: %s %s %s %s", sdp[3], sdp[2],sdp[1],sdp[0])
res=UpqDB().query("SELECT * FROM file f WHERE name='%s'" % sdp[3])
if res:
row=res.first()
self.logger.error("a file with this name already exists, fid=%s, sdp=%s" % (row['fid'], row['sdp']))
def check(self):
results = UpqDB().query("SELECT fid, filename, path FROM file WHERE fid=%d " % int(self.jobdata['fid']))
res=results.first()
filename = os.path.join(UpqConfig().paths['files'], res['path'], res['filename'])
if not os.path.exists(filename):
self.logger.error("file %d doesn't exist: %s"%(res['fid'], filename))
return False
self.enqueue_job()
return True
def getCID(self, category):
if category in self.cats:
return self.cats[category]
res = UpqDB().query("SELECT cid from categories WHERE name='%s'" % (category))
try:
self.cats[category]=res.first()[0] # cache result
except:
self.logger.error("Invalid category: %s" % category)
return self.cats[category]
def getCid(self, name):
result = UpqDB().query("SELECT cid from categories WHERE name='%s'" %
name)
res = result.first()
if res:
cid = res['cid']
else:
cid = UpqDB().insert("categories", {"name": name})
return cid
def getCID(self, category):
if category in self.cats:
return self.cats[category]
res = UpqDB().query("SELECT cid from categories WHERE name='%s'" %
(category))
try:
self.cats[category] = res.first()[0] # cache result
except:
self.logger.error("Invalid category: %s" % category)
return self.cats[category]
def check(self):
if not 'fid' in self.jobdata:
self.msg("fid not specified")
return False
results=UpqDB().query("SELECT * FROM file WHERE fid=%s and status=1" % (int(self.jobdata['fid'])))
res=results.first()
if res == None:
self.msg("fid not found")
return False
id=self.enqueue_job()
return True
def check(self):
if not "fid" in self.jobdata:
self.msg("fid not specified")
return False
results = UpqDB().query("SELECT * FROM file WHERE fid=%s and status=1" % (int(self.jobdata["fid"])))
res = results.first()
if res == None:
self.msg("fid not found")
return False
id = self.enqueue_job()
return True
def check(self):
results = UpqDB().query(
"SELECT fid, filename, path FROM file WHERE fid=%d " %
int(self.jobdata['fid']))
res = results.first()
filename = os.path.join(UpqConfig().paths['files'], res['path'],
res['filename'])
if not os.path.exists(filename):
self.logger.error("file %d doesn't exist: %s" %
(res['fid'], filename))
return False
self.enqueue_job()
return True
def run(self):
dled = {}
url = self.prefix + '/list.php'
#print self.getlobbyversion()
f = my_download().open(url)
data = json.loads(str(f.read()))
res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" % self.prefix)
mid = res.first()[0]
for row in data:
self.update(row, mid)
#delete files that wheren't updated this run (means removed from mirror)
UpqDB().query("DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s" %(mid))
urllib.urlcleanup()
return True
def update(self, data, mid):
"""
data is an array with
md5
filectime
version
branch
filesize
os
path
"""
if data['version'] == "testing":
return
filename = self.escape(data['path'][data['path'].rfind("/") + 1:])
category = "engine_" + data['os']
branch = data['branch']
version = data['version']
if not data['branch'] in ('master'):
version = data['version'] + ' ' + data['branch']
url = self.prefix + '/' + data['path']
cid = self.getCID(category)
#print "%s %s %s %s" % (filename, version, category, url)
try:
fid = UpqDB().insert(
"file",
{
"filename": filename,
"name": "spring",
"version": version,
"cid": cid,
"md5": data['md5'],
"timestamp": datetime.datetime.fromtimestamp(
data['filectime']),
#"timestamp": data['filectime'],
"size": data['filesize'],
"status": 1
})
except UpqDBIntegrityError, e:
try:
res = UpqDB().query(
"SELECT fid from file WHERE version='%s' and cid=%s" %
(version, cid))
fid = res.first()[0]
UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s" %
(data['md5'], fid))
except Exception, e:
self.logger.error("Error %s %s %s", version, cid, e)
return
def check(self):
if 'file' in self.jobdata :
self.jobdata['filename']=os.path.basename(self.jobdata['file'])
result=UpqDB().query("SELECT * from file where filename LIKE '%s'" % (self.jobdata['filename']))
res=result.first()
if res!=None:
self.jobdata['fid']=res['fid']
self.msg("File %s already exists: fid: %s"%(self.jobdata['file'], res['fid']))
if not os.access(self.jobdata['file'], os.R_OK):
self.msg("Can't access %s"% self.jobdata['file'])
return False
elif not 'fid' in self.jobdata:
self.msg("Either file or fid has to be set!")
return False
self.enqueue_job()
return True
def run(self):
fid=int(self.jobdata['fid'])
results=UpqDB().query("SELECT filename, path, sdp FROM file WHERE fid=%d AND status=1" % fid)
res=results.first()
#filename of the archive to be scanned
filename=res['filename'] # filename only (no path info)
absfilename=os.path.join(UpqConfig().paths['files'], res['path'], res['filename']) # absolute filename
torrent=os.path.join(UpqConfig().paths['metadata'], res['sdp']+ ".torrent" )
if not os.path.exists(absfilename):
self.msg("File doesn't exist: %s" %(absfilename))
return False
res=self.create_torrent(absfilename, torrent)
if res:
UpqDB().query("UPDATE file SET torrent=1 WHERE fid=%s" %(fid))
return res
def run(self):
fid = int(self.jobdata["fid"])
results = UpqDB().query("SELECT filename, path, sdp FROM file WHERE fid=%d AND status=1" % fid)
res = results.first()
# filename of the archive to be scanned
filename = res["filename"] # filename only (no path info)
absfilename = os.path.join(UpqConfig().paths["files"], res["path"], res["filename"]) # absolute filename
torrent = os.path.join(UpqConfig().paths["metadata"], res["sdp"] + ".torrent")
if not os.path.exists(absfilename):
self.msg("File doesn't exist: %s" % (absfilename))
return False
res = self.create_torrent(absfilename, torrent)
if res:
UpqDB().query("UPDATE file SET torrent=1 WHERE fid=%s" % (fid))
return res
def run(self):
dled = {}
url = self.prefix + '/list.php'
#print self.getlobbyversion()
f = my_download().open(url)
data = json.loads(str(f.read()))
res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" %
self.prefix)
mid = res.first()[0]
for row in data:
self.update(row, mid)
#delete files that wheren't updated this run (means removed from mirror)
UpqDB().query(
"DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s"
% (mid))
urllib.urlcleanup()
return True
def run(self):
"""
params:
filepath: absoulte path of file
filename: filename
uid: uid
or:
fid
"""
if 'fid' in self.jobdata: # file already exists in db, set filename
result=UpqDB().query("SELECT * from file where fid=%s"% (self.jobdata['fid']))
res=result.first()
if res==None:
self.msg("fid not found in db!")
return False
self.msg("File already known, Filename: %s Size: %d" % (res['filename'], res['size']))
return True
def deleteOldFiles(self):
"""
deletes files from ftpmirror when:
file is older than 100 days
it is known in rapid
it is not a stable|test tag (no number at the end in tag)
"""
results = UpqDB().query(
"SELECT m.mid, ftp_url, ftp_user, ftp_pass, ftp_dir, ftp_port, ftp_passive, ftp_ssl \
FROM mirror as m \
WHERE m.status=1")
for mirror in results:
results2 = UpqDB().query("SELECT f.fid,tag FROM `file` f \
LEFT JOIN tag t ON f.fid=t.fid \
WHERE (t.fid>0) \
AND f.status=1 \
AND timestamp < NOW() - INTERVAL 100 DAY \
GROUP BY f.fid HAVING count(f.fid) = 1")
ftp = None
for filetodel in results2:
if not ftp:
try:
ftp = self.ftpconnect(
mirror['ftp_url'], mirror['ftp_port'],
mirror['ftp_user'], mirror['ftp_pass'],
mirror['ftp_passive'], mirror['ftp_dir'],
mirror['ftp_ssl'])
except Exception as e:
self.logger.error("Couldn't connect to %s: %s",
mirror['ftp_url'], e)
break
res2 = UpqDB().query(
"SELECT * FROM mirror_file WHERE fid = %d AND mid = %d AND status=1"
% (filetodel['fid'], mirror['mid']))
curfile = res2.first()
if curfile and filetodel['tag'][-1].isdigit():
try:
self.logger.error("deleting %s" % (curfile['path']))
ftp.delete(curfile['path'])
except:
pass
UpqDB().query(
"UPDATE mirror_file SET status=4 WHERE mfid = %d" %
(int(curfile['mfid'])))
if ftp:
ftp.close()
def run(self):
"""
params:
filepath: absoulte path of file
filename: filename
uid: uid
or:
fid
"""
if 'fid' in self.jobdata: # file already exists in db, set filename
result = UpqDB().query("SELECT * from file where fid=%s" %
(self.jobdata['fid']))
res = result.first()
if res == None:
self.msg("fid not found in db!")
return False
self.msg("File already known, Filename: %s Size: %d" %
(res['filename'], res['size']))
return True
def check(self):
if 'file' in self.jobdata:
self.jobdata['filename'] = os.path.basename(self.jobdata['file'])
result = UpqDB().query(
"SELECT * from file where filename LIKE '%s'" %
(self.jobdata['filename']))
res = result.first()
if res != None:
self.jobdata['fid'] = res['fid']
self.msg("File %s already exists: fid: %s" %
(self.jobdata['file'], res['fid']))
if not os.access(self.jobdata['file'], os.R_OK):
self.msg("Can't access %s" % self.jobdata['file'])
return False
elif not 'fid' in self.jobdata:
self.msg("Either file or fid has to be set!")
return False
self.enqueue_job()
return True
def update(self, data, mid):
"""
data is an array with
md5
filectime
version
branch
filesize
os
path
"""
if data['version'] == "testing":
return
filename = self.escape(data['path'][data['path'].rfind("/")+1:])
category = "engine_" + data['os']
branch = data['branch']
version = data['version']
if not data['branch'] in ('master'):
version = data['version'] + ' ' + data['branch']
url = self.prefix +'/' + data['path']
cid = self.getCID(category)
#print "%s %s %s %s" % (filename, version, category, url)
try:
fid = UpqDB().insert("file", {
"filename" : filename,
"name": "spring",
"version": version,
"cid" : cid,
"md5" : data['md5'],
"timestamp": datetime.datetime.fromtimestamp(data['filectime']),
#"timestamp": data['filectime'],
"size": data['filesize'],
"status": 1 })
except UpqDBIntegrityError, e:
try:
res = UpqDB().query("SELECT fid from file WHERE version='%s' and cid=%s" % (version, cid))
fid = res.first()[0]
UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s"% (data['md5'], fid))
except Exception, e:
self.logger.error("Error %s %s %s", version, cid, e)
return
def check(self):
if 'file' in self.jobdata:
if not os.path.exists(self.jobdata['file']):
self.logger.error("File %s doesn't exist" % self.jobdata['file'])
return False
elif 'fid' in self.jobdata:
results=UpqDB().query("SELECT filename, path, status FROM file WHERE fid=%s" % (int(self.jobdata['fid'])))
res=results.first()
if not res:
self.logger.error("Fid not found in db")
return False
prefix=self.getPathByStatus(res['status'])
self.jobdata['file']=os.path.join(prefix, res['path'], res['filename'] )
if not os.path.exists(self.jobdata['file']):
self.logger.error("filepath from db doesn't exist %s" %(self.jobdata['file']))
return False
else:
self.logger.error("Either fid or file has to be set")
return False
self.enqueue_job()
return True
def getmetadata(self, fid):
"""
returns the metadata, that is required for springfiles in a struct
required data:
link to images
mirror urls
file size
md5
"""
data={}
data['mirror']=[]
results=UpqDB().query("SELECT CONCAT(m.url_prefix, f.path) as url FROM mirror_file f LEFT JOIN mirror m ON f.mid=m.mid WHERE f.fid=%d" % (fid))
for res in results:
data['mirror'].append(res['url'])
results=UpqDB().query("SELECT f.filename, f.size, f.timestamp, f.md5, f.name, f.version, c.name as category, f.metadata FROM file f LEFT JOIN categories c ON f.cid=c.cid WHERE f.fid=%d" %(fid))
res=results.first()
for value in ["filename", "size", "timestamp", "md5", "name", "version", "category"]:
data[value]=res[value]
if res['metadata']!=None and len(res['metadata'])>0:
data['metadata']=json.loads(res['metadata'])
else:
data['metadata']=[]
return data
def run(self):
dled = {}
url = self.prefix + '/list.php'
filename = "/tmp/sprinvers.json"
if not self.DownloadFile(url, filename):
self.logger.info("list.php wasn't changed")
return True
with open(filename, "r") as f:
data = json.loads(str(f.read()))
res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" %
self.prefix)
mid = res.first()[0]
assert (mid > 0)
for row in data:
self.update(row, mid)
#delete files that wheren't updated this run (means removed from mirror)
UpqDB().query(
"DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s"
% (mid))
return True
def insertData(self, data, filename, hashes):
metadata = data.copy()
del metadata['Depends'] #remove redundant entries
del metadata['sdp']
del metadata['Version']
del metadata['Name']
self.logger.debug(metadata)
metadata = json.dumps(metadata)
if 'fid' in self.jobdata: # detect already existing files
fid = self.jobdata['fid']
else:
results = UpqDB().query(
"SELECT fid FROM file WHERE sdp='%s' or md5='%s'" %
(data['sdp'], hashes['md5']))
res = results.first()
if res:
fid = res['fid']
else:
fid = 0
if fid <= 0:
fid = UpqDB().insert(
"file",
{
"name": self.escape(data['Name']),
"version": self.escape(data['Version']),
"sdp": data['sdp'],
"cid": self.getCid(data['Type']),
"metadata": metadata,
"uid": 0,
"path": data["path"],
"filename": os.path.basename(filename),
"timestamp": UpqDB().now(), #fixme: use file timestamp
"size": os.path.getsize(filename),
"status": 1,
"md5": hashes["md5"],
"sha1": hashes["sha1"],
"sha256": hashes["sha256"],
})
else:
UpqDB().query(
"UPDATE file SET name='%s', version='%s', sdp='%s', cid=%s, metadata='%s', md5='%s', sha1='%s', sha256='%s', status=1 WHERE fid=%s"
% (self.escape(data['Name']), data['Version'], data['sdp'],
self.getCid(data['Type']), metadata, hashes["md5"],
hashes["sha1"], hashes["sha256"], fid))
# remove already existing depends
UpqDB().query("DELETE FROM file_depends WHERE fid = %s" % (fid))
for depend in data['Depends']:
res = UpqDB().query(
"SELECT fid FROM file WHERE CONCAT(name,' ',version)='%s'" %
(depend))
row = res.first()
if not row:
id = 0
else:
id = row['fid']
try:
UpqDB().insert("file_depends", {
"fid": fid,
"depends_string": depend,
"depends_fid": id
})
self.msg("Added %s '%s' version '%s' to the mirror-system" %
(data['Type'], data['Name'], data['Version']))
except UpqDBIntegrityError:
pass
self.msg("Updated %s '%s' version '%s' sdp '%s' in the mirror-system" %
(data['Type'], data['Name'], data['Version'], data['sdp']))
return fid
def update(self, data, mid):
"""
data is an array with
md5
filectime
version
branch
filesize
os
path
"""
if data['version'] == "testing":
return
filename = self.escape(data['path'][data['path'].rfind("/") + 1:])
category = "engine_" + data['os']
branch = data['branch']
version = data['version']
if not data['branch'] in ('master'):
version = data['version'] + ' ' + data['branch']
url = self.prefix + '/' + data['path']
cid = self.getCID(category)
#print "%s %s %s %s" % (filename, version, category, url)
res = UpqDB().query(
"SELECT fid from file WHERE version='%s' and cid=%s" %
(version, cid))
fid = res.first()
if fid:
fid = fid[0]
assert (fid > 0)
UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s" %
(data['md5'], fid))
else:
fid = UpqDB().insert(
"file",
{
"filename": filename,
"name": "spring",
"version": version,
"cid": cid,
"md5": data['md5'],
"timestamp": datetime.datetime.fromtimestamp(
data['filectime']),
#"timestamp": data['filectime'],
"size": data['filesize'],
"status": 1
})
res = UpqDB().query(
"SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s" %
(mid, fid))
mfid = res.first()
if mfid:
mfid = mfid[0]
assert (mid > 0)
UpqDB().query(
"UPDATE mirror_file SET lastcheck=NOW() WHERE mfid = %s" %
(mfid))
else:
relpath = self.escape(url[len(self.prefix) + 1:])
mfid = UpqDB().insert(
"mirror_file", {
"mid": mid,
"path": relpath,
"status": 1,
"fid": fid,
"lastcheck": UpqDB().now()
})
return
relpath = self.escape(url[len(self.prefix) + 1:])
try:
id = UpqDB().insert(
"mirror_file", {
"mid": mid,
"path": relpath,
"status": 1,
"fid": fid,
"lastcheck": UpqDB().now()
})
except UpqDBIntegrityError:
res = UpqDB().query(
"SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s" %
(mid, fid))
id = res.first()[0]
UpqDB().query(
"UPDATE mirror_file SET lastcheck=NOW() WHERE mfid = %s" %
(id))
def run(self):
dled = {}
url = self.prefix + '/list.php'
#print self.getlobbyversion()
f = my_download().open(url)
data = json.loads(str(f.read()))
res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" %
self.prefix)
mid = res.first()[0]
for row in data:
self.update(row, mid)
def run(self):
fid = int(self.jobdata['fid'])
results = UpqDB().query(
"SELECT filename, path, size, md5 FROM file where fid=%d AND status=1"
% (fid))
if results.rowcount != 1:
self.msg("Wrong result count with fid %d" % fid)
return False
res = results.first()
srcfilename = os.path.join(UpqConfig().paths['files'], res['path'],
res['filename'])
dstfilename = os.path.join(res['path'], res['filename'])
filesize = res['size']
md5 = res['md5']
#uploads a fid to all mirrors
results = UpqDB().query(
"SELECT m.mid, ftp_url, ftp_user, ftp_pass, ftp_dir, ftp_port, ftp_passive, ftp_ssl \
from file f \
left join mirror m on m.status=f.status \
where f.fid=%d and f.status=1 and m.status=1 \
and m.mid not in (select mid from mirror_file where fid=%d and status = 1)"
% (fid, fid))
uploadcount = 0
for res in results:
try:
ftp = self.ftpconnect(res['ftp_url'], res['ftp_port'],
res['ftp_user'], res['ftp_pass'],
res['ftp_passive'], res['ftp_dir'],
res['ftp_ssl'])
except Exception as e:
self.logger.error("Couldn't connect to the ftp server: %s", e)
continue
if not os.path.isfile(srcfilename):
self.msg("File doesn't exist: " + srcfilename)
return False
try:
f = open(srcfilename, "rb")
dstdir = os.path.dirname(dstfilename)
try:
self.logger.debug("cd into " + dstdir)
ftp.cwd(dstdir)
except:
try:
self.logger.debug("mkdir " + dstdir)
ftp.mkd(dstdir)
self.logger.debug("cwd " + dstdir)
ftp.cwd(dstdir)
except:
self.logger.error(
"couldn't cd/mkdir %s, skipping upload " %
(dstdir))
continue
self.logger.info("uploading %s to %s" %
(os.path.basename(dstfilename),
(res['ftp_url'])))
ftp.storbinary('STOR ' + os.path.basename(dstfilename), f)
ftp.quit()
f.close()
try: #upload succeed, mark in db as uploaded
id = UpqDB().insert(
"mirror_file", {
"mid": res['mid'],
"fid": fid,
"path": dstfilename,
"status": 1
})
except UpqDBIntegrityError:
res = UpqDB().query(
"SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s"
% (res['mid'], fid))
id = res.first()
self.logger.info("file already uploaded: mfid=%d" % (id))
UpqDB().query(
"UPDATE mirror_file SET status=1 path='%s' WHERE mfid=%s"
% (dstfilename, id))
self.logger.debug("inserted into db as %d", id)
self.enqueue_newjob("verify_remote_file", {"mfid": id})
except ftplib.all_errors as e:
self.logger.error("Ftp-Error (%s) %s failed %s" %
((res['ftp_url'], srcfilename, e)))
except Exception as e:
self.logger.error("Upload (%s) %s failed %s" %
((res['ftp_url'], srcfilename, e)))
return False
uploadcount += 1
self.msg("Uploaded to %d mirrors." % (uploadcount))
self.deleteOldFiles()
return True
UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s"% (data['md5'], fid))
except Exception, e:
self.logger.error("Error %s %s %s", version, cid, e)
return
relpath = self.escape(url[len(self.prefix)+1:])
try:
id = UpqDB().insert("mirror_file", {
"mid" : mid,
"path": relpath,
"status": 1,
"fid": fid,
"lastcheck": UpqDB().now()
})
except UpqDBIntegrityError:
res = UpqDB().query("SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s" % (mid, fid))
id = res.first()[0]
UpqDB().query("UPDATE mirror_file SET lastcheck=NOW() WHERE mfid = %s"% (id))
def run(self):
dled = {}
url = self.prefix + '/list.php'
#print self.getlobbyversion()
f = my_download().open(url)
data = json.loads(str(f.read()))
res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" % self.prefix)
mid = res.first()[0]
for row in data:
self.update(row, mid)
#delete files that wheren't updated this run (means removed from mirror)
UpqDB().query("DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s" %(mid))
urllib.urlcleanup()
