def getmetadata(self, fid): """ returns the metadata, that is required for springfiles in a struct required data: link to images mirror urls file size md5 """ data = {} data['mirror'] = [] results = UpqDB().query( "SELECT CONCAT(m.url_prefix, f.path) as url FROM mirror_file f LEFT JOIN mirror m ON f.mid=m.mid WHERE f.fid=%d" % (fid)) for res in results: data['mirror'].append(res['url']) results = UpqDB().query( "SELECT f.filename, f.size, f.timestamp, f.md5, f.name, f.version, c.name as category, f.metadata FROM file f LEFT JOIN categories c ON f.cid=c.cid WHERE f.fid=%d" % (fid)) res = results.first() for value in [ "filename", "size", "timestamp", "md5", "name", "version", "category" ]: data[value] = res[value] if res['metadata'] != None and len(res['metadata']) > 0: data['metadata'] = json.loads(res['metadata']) else: data['metadata'] = [] return data
def run(self): """ class Hash must be initialized with fileid! """ fid = int(self.jobdata['fid']) results = UpqDB().query("SELECT filename, path, md5, sha1, sha256 FROM file WHERE fid=%d " % int(fid)) res=results.first() filename = os.path.join(UpqConfig().paths['files'], res['path'], res['filename']) if not os.path.exists(filename): self.msg("File %s doesn't exist" % (filename)) return False hashes = self.hash(filename) if res['md5']!=None and res['md5']!=hashes['md5']: self.logger.error("md5 missmatch") return False if res['sha1']!=None and res['sha1']!=hashes['sha1']: self.logger.error("sha1 missmatch") return False if res['sha256']!=None and res['sha256']!=hashes['sha256']: self.logger.error("sha256 missmatch") return False UpqDB().query("UPDATE file set md5='%s', sha1='%s', sha256='%s' WHERE fid=%d" % (hashes['md5'], hashes['sha1'], hashes['sha256'], fid)) self.msg("md5: %s sha1: %s sha256: %s" % (hashes['md5'], hashes['sha1'], hashes['sha256'])) return True
def normalizeFilename(self, srcfile, fid, name, version): """ normalize filename + renames file + updates filename in database """ name=name.lower() if len(version)>0: name = str(name[:200] +"-" + version.lower())[:255] _, extension = os.path.splitext(srcfile) name += extension res="" for c in name: if c in "abcdefghijklmnopqrstuvwxyz-_.01234567890": res+=c else: res+="_" dstfile=os.path.join(os.path.dirname(srcfile), res) if srcfile!=dstfile: results=UpqDB().query("SELECT fid FROM file WHERE BINARY filename='%s' AND status=1" % (res)) row=results.first() if row or os.path.exists(dstfile): self.logger.error("Error renaming file: %s to %s already exists, deleting source + using dst!" % (srcfile, dstfile)) os.remove(srcfile) else: shutil.move(srcfile, dstfile) UpqDB().query("UPDATE file SET filename='%s' WHERE fid=%s" %(res, fid)) self.logger.info("Normalized filename to %s "%(dstfile)) return dstfile
def check(self): if 'file' in self.jobdata: if not os.path.exists(self.jobdata['file']): self.logger.error("File %s doesn't exist" % self.jobdata['file']) return False elif 'fid' in self.jobdata: results = UpqDB().query( "SELECT filename, path, status FROM file WHERE fid=%s" % (int(self.jobdata['fid']))) res = results.first() if not res: self.logger.error("Fid not found in db") return False prefix = self.getPathByStatus(res['status']) self.jobdata['file'] = os.path.join(prefix, res['path'], res['filename']) if not os.path.exists(self.jobdata['file']): self.logger.error("filepath from db doesn't exist %s" % (self.jobdata['file'])) return False else: self.logger.error("Either fid or file has to be set") return False self.enqueue_job() return True
def run(self): """ class Hash must be initialized with fileid! """ fid = int(self.jobdata['fid']) results = UpqDB().query( "SELECT filename, path, md5, sha1, sha256 FROM file WHERE fid=%d " % int(fid)) res = results.first() filename = os.path.join(UpqConfig().paths['files'], res['path'], res['filename']) if not os.path.exists(filename): self.msg("File %s doesn't exist" % (filename)) return False hashes = self.hash(filename) if res['md5'] != None and res['md5'] != hashes['md5']: self.logger.error("md5 missmatch") return False if res['sha1'] != None and res['sha1'] != hashes['sha1']: self.logger.error("sha1 missmatch") return False if res['sha256'] != None and res['sha256'] != hashes['sha256']: self.logger.error("sha256 missmatch") return False UpqDB().query( "UPDATE file set md5='%s', sha1='%s', sha256='%s' WHERE fid=%d" % (hashes['md5'], hashes['sha1'], hashes['sha256'], fid)) self.msg("md5: %s sha1: %s sha256: %s" % (hashes['md5'], hashes['sha1'], hashes['sha256'])) return True
def getCid(self, name): result=UpqDB().query("SELECT cid from categories WHERE name='%s'" % name) res=result.first() if res: cid=res['cid'] else: cid=UpqDB().insert("categories", {"name": name}) return cid
def getCid(self, name): result = UpqDB().query("SELECT cid from categories WHERE name='%s'" % name) res = result.first() if res: cid = res['cid'] else: cid = UpqDB().insert("categories", {"name": name}) return cid
def getCID(self, category): if category in self.cats: return self.cats[category] res = UpqDB().query("SELECT cid from categories WHERE name='%s'" % (category)) try: self.cats[category]=res.first()[0] # cache result except: self.logger.error("Invalid category: %s" % category) return self.cats[category]
def check(self): results = UpqDB().query("SELECT fid, filename, path FROM file WHERE fid=%d " % int(self.jobdata['fid'])) res=results.first() filename = os.path.join(UpqConfig().paths['files'], res['path'], res['filename']) if not os.path.exists(filename): self.logger.error("file %d doesn't exist: %s"%(res['fid'], filename)) return False self.enqueue_job() return True
def insertData(self, data, filename): metadata=data.copy() del metadata['Depends'] #remove redundant entries del metadata['sdp'] del metadata['Version'] del metadata['Name'] metadata=json.dumps(metadata) if 'fid' in self.jobdata: # detect already existing files fid=self.jobdata['fid'] else: results=UpqDB().query("SELECT fid FROM file WHERE sdp='%s'"% (data['sdp'])) res=results.first() if res: fid=res['fid'] else: fid=0 if fid<=0: fid=UpqDB().insert("file", { "name": self.escape(data['Name']), "version": self.escape(data['Version']), "sdp": data['sdp'], "cid": self.getCid(data['Type']), "metadata": metadata, "uid": 0, "path": "", "filename": filename, "timestamp": UpqDB().now(), #fixme: use file timestamp "size": os.path.getsize(filename), "status": 1, }) else: UpqDB().query("UPDATE file SET name='%s', version='%s', sdp='%s', cid=%s, metadata='%s' WHERE fid=%s" %( self.escape(data['Name']), data['Version'], data['sdp'], self.getCid(data['Type']), metadata, fid )) # remove already existing depends UpqDB().query("DELETE FROM file_depends WHERE fid = %s" % (fid) ) for depend in data['Depends']: res=UpqDB().query("SELECT fid FROM file WHERE CONCAT(name,' ',version)='%s'" % (depend)) row=res.first() if not row: id=0 else: id=row['fid'] try: UpqDB().insert("file_depends", {"fid":fid, "depends_string": depend, "depends_fid": id}) self.msg("Added %s '%s' version '%s' to the mirror-system" % (data['Type'], data['Name'], data['Version'])) except UpqDBIntegrityError: pass self.msg("Updated %s '%s' version '%s' sdp '%s' in the mirror-system" % (data['Type'], data['Name'], data['Version'], data['sdp'])) return fid
def getCID(self, category): if category in self.cats: return self.cats[category] res = UpqDB().query("SELECT cid from categories WHERE name='%s'" % (category)) try: self.cats[category] = res.first()[0] # cache result except: self.logger.error("Invalid category: %s" % category) return self.cats[category]
def check(self): if not "fid" in self.jobdata: self.msg("fid not specified") return False results = UpqDB().query("SELECT * FROM file WHERE fid=%s and status=1" % (int(self.jobdata["fid"]))) res = results.first() if res == None: self.msg("fid not found") return False id = self.enqueue_job() return True
def check(self): if not 'fid' in self.jobdata: self.msg("fid not specified") return False results=UpqDB().query("SELECT * FROM file WHERE fid=%s and status=1" % (int(self.jobdata['fid']))) res=results.first() if res == None: self.msg("fid not found") return False id=self.enqueue_job() return True
def task_done(self, job): # update job state in DB if job.result: result = 0 else: result = 1 if len(job.msgstr) > 255: #limit string length job.msgstr = job.msgstr[:255] query = "UPDATE upqueue SET status=0, end_time=NOW(), result_msg='%s' WHERE jobid = %s" % ( UpqDB().escape(job.msgstr), job.jobid) UpqDB().query(query) super(DBQueue, self).task_done()
def check(self): results = UpqDB().query( "SELECT fid, filename, path FROM file WHERE fid=%d " % int(self.jobdata['fid'])) res = results.first() filename = os.path.join(UpqConfig().paths['files'], res['path'], res['filename']) if not os.path.exists(filename): self.logger.error("file %d doesn't exist: %s" % (res['fid'], filename)) return False self.enqueue_job() return True
def run(self): results = UpqDB().query("SELECT count(mid) from mirror") count = res.first()[0] results = UpqDB().query( "SELECT f.mfid FROM mirror_file f LEFT JOIN mirror m ON f.mid=m.mid \ WHERE m.status=1 \ AND ( f.lastcheck < DATE_SUB(NOW(), INTERVAL 1 MONTH) \ OR f.lastcheck is null ) \ ORDER BY f.lastcheck ASC \ LIMIT 0,30") for res in results: self.enqueue_newjob("verify_remote_file", {"mfid": res["mfid"]}) sleep(10) return True
def put(self, job, block=True, timeout=None): if job.jobid == -1: # add job to DB in state "new" jobdata = json.dumps(job.jobdata, -1) ret = UpqDB().insert( "upqueue", { 'jobname': job.jobname, 'status': 2, 'jobdata': jobdata, 'ctime': UpqDB().now() }) job.jobid = ret #set jobid super(DBQueue, self).put(job, block, timeout) return job.jobid
def run(self): dled = {} url = self.prefix + '/list.php' #print self.getlobbyversion() f = my_download().open(url) data = json.loads(str(f.read())) res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" % self.prefix) mid = res.first()[0] for row in data: self.update(row, mid) #delete files that wheren't updated this run (means removed from mirror) UpqDB().query("DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s" %(mid)) urllib.urlcleanup() return True
def update(self, data, mid): """ data is an array with md5 filectime version branch filesize os path """ if data['version'] == "testing": return filename = self.escape(data['path'][data['path'].rfind("/") + 1:]) category = "engine_" + data['os'] branch = data['branch'] version = data['version'] if not data['branch'] in ('master'): version = data['version'] + ' ' + data['branch'] url = self.prefix + '/' + data['path'] cid = self.getCID(category) #print "%s %s %s %s" % (filename, version, category, url) try: fid = UpqDB().insert( "file", { "filename": filename, "name": "spring", "version": version, "cid": cid, "md5": data['md5'], "timestamp": datetime.datetime.fromtimestamp( data['filectime']), #"timestamp": data['filectime'], "size": data['filesize'], "status": 1 }) except UpqDBIntegrityError, e: try: res = UpqDB().query( "SELECT fid from file WHERE version='%s' and cid=%s" % (version, cid)) fid = res.first()[0] UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s" % (data['md5'], fid)) except Exception, e: self.logger.error("Error %s %s %s", version, cid, e) return
def run(self): fid=int(self.jobdata['fid']) results=UpqDB().query("SELECT filename, path, sdp FROM file WHERE fid=%d AND status=1" % fid) res=results.first() #filename of the archive to be scanned filename=res['filename'] # filename only (no path info) absfilename=os.path.join(UpqConfig().paths['files'], res['path'], res['filename']) # absolute filename torrent=os.path.join(UpqConfig().paths['metadata'], res['sdp']+ ".torrent" ) if not os.path.exists(absfilename): self.msg("File doesn't exist: %s" %(absfilename)) return False res=self.create_torrent(absfilename, torrent) if res: UpqDB().query("UPDATE file SET torrent=1 WHERE fid=%s" %(fid)) return res
def run(self): fid = int(self.jobdata["fid"]) results = UpqDB().query("SELECT filename, path, sdp FROM file WHERE fid=%d AND status=1" % fid) res = results.first() # filename of the archive to be scanned filename = res["filename"] # filename only (no path info) absfilename = os.path.join(UpqConfig().paths["files"], res["path"], res["filename"]) # absolute filename torrent = os.path.join(UpqConfig().paths["metadata"], res["sdp"] + ".torrent") if not os.path.exists(absfilename): self.msg("File doesn't exist: %s" % (absfilename)) return False res = self.create_torrent(absfilename, torrent) if res: UpqDB().query("UPDATE file SET torrent=1 WHERE fid=%s" % (fid)) return res
def check(self): if 'file' in self.jobdata : self.jobdata['filename']=os.path.basename(self.jobdata['file']) result=UpqDB().query("SELECT * from file where filename LIKE '%s'" % (self.jobdata['filename'])) res=result.first() if res!=None: self.jobdata['fid']=res['fid'] self.msg("File %s already exists: fid: %s"%(self.jobdata['file'], res['fid'])) if not os.access(self.jobdata['file'], os.R_OK): self.msg("Can't access %s"% self.jobdata['file']) return False elif not 'fid' in self.jobdata: self.msg("Either file or fid has to be set!") return False self.enqueue_job() return True
def get(self, block=True, timeout=None): job = super(DBQueue, self).get(block, timeout) # update job state in DB query = "UPDATE upqueue SET status=1, start_time=NOW() WHERE jobid = %d" % ( job.jobid) UpqDB().query(query) return job
def run(self): dled = {} url = self.prefix + '/list.php' #print self.getlobbyversion() f = my_download().open(url) data = json.loads(str(f.read())) res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" % self.prefix) mid = res.first()[0] for row in data: self.update(row, mid) #delete files that wheren't updated this run (means removed from mirror) UpqDB().query( "DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s" % (mid)) urllib.urlcleanup() return True
def run(self): """ params: filepath: absoulte path of file filename: filename uid: uid or: fid """ if 'fid' in self.jobdata: # file already exists in db, set filename result=UpqDB().query("SELECT * from file where fid=%s"% (self.jobdata['fid'])) res=result.first() if res==None: self.msg("fid not found in db!") return False self.msg("File already known, Filename: %s Size: %d" % (res['filename'], res['size'])) return True
def UpdateSDP(self, sdp): """ values = { "tag"=sdp[0], "md5"=sdp[1], "depends"=sdp[2], "name"=sdp[3], } """ #check if file is already known res=UpqDB().query("SELECT f.fid FROM file f \ LEFT JOIN tag t ON t.fid=f.fid \ WHERE sdp='%s'" % (sdp[1])) row=res.first() if row: #file is already known #delete tag from existing files UpqDB().query("DELETE FROM tag WHERE tag='%s'" % (sdp[0])) #insert updated tag UpqDB().query("INSERT INTO tag (fid, tag) VALUES (%s, '%s')" % (row['fid'], sdp[0])) #self.logger.info("updated %s %s %s %s",sdp[3],sdp[2],sdp[1],sdp[0]) return if not sdp[3]: #without a name, we can't do anything! return cid = self.getCid("game") try: fid = UpqDB().insert("file", { "filename" : sdp[3] + " (not available as sdz)", "name": sdp[3], "cid" : cid, "md5" : sdp[1], "sdp" : sdp[1], "size" : 0, "status" : 4, # special status for files that can only be downloaded via rapid "uid" : 0, "path" : "", }) UpqDB().query("INSERT INTO tag (fid, tag) VALUES (%s, '%s')" % (fid, sdp[0])) #self.logger.info("inserted %s %s %s %s",sdp[3],sdp[2],sdp[1],sdp[0]) except Exception as e: self.logger.error(str(e)) self.logger.error("Error from sdp: %s %s %s %s", sdp[3], sdp[2],sdp[1],sdp[0]) res=UpqDB().query("SELECT * FROM file f WHERE name='%s'" % sdp[3]) if res: row=res.first() self.logger.error("a file with this name already exists, fid=%s, sdp=%s" % (row['fid'], row['sdp']))
def ExtractMetadata(self, usync, archiveh, filename, filepath, metadatapath, hashes): filelist = self.getFileList(usync, archiveh) sdp = self.getSDPName(usync, archiveh) idx = self.getMapIdx(usync, filename) if idx >= 0: #file is map archivepath = usync.GetArchivePath( filename.encode()).decode() + filename springname = usync.GetMapName(idx).decode() data = self.getMapData(usync, filename, idx, archiveh, springname) data['mapimages'] = self.dumpmap(usync, springname, metadatapath, filename, idx) data['path'] = self.jobcfg['maps-path'] else: # file is a game idx = self.getGameIdx(usync, filename) if idx < 0: self.logger.error("Invalid file detected: %s %s %s" % (filename, usync.GetNextError(), idx)) return False self.logger.debug("Extracting data from " + filename) archivepath = usync.GetArchivePath(filename) + filename gamearchivecount = usync.GetPrimaryModArchiveCount( idx) # initialization for GetPrimaryModArchiveList() data = self.getGameData(usync, idx, gamearchivecount, archivepath, archiveh) data['path'] = self.jobcfg['games-path'] if (sdp == "") or (data['Name'] == "" ): #mark as broken because sdp / name is missing self.logger.error("Couldn't get name / filename") return False data['splash'] = self.createSplashImages(usync, archiveh, filelist) _, extension = os.path.splitext(filename) moveto = os.path.join( self.getPathByStatus(1), data['path'], self.GetNormalizedFilename(data['Name'], data['Version'], extension)) self.jobdata['file'] = moveto assert (moveto != filepath) assert (len(moveto) > 0) if not self.movefile(filepath, moveto): self.logger.error("Couldn't move file %s -> %s" % (filepath, moveto)) return False try: data['sdp'] = sdp self.jobdata['fid'] = self.insertData(data, moveto, hashes) except UpqDBIntegrityError: self.logger.error("Duplicate file detected: %s %s %s" % (filename, data['Name'], data['Version'])) return False if self.create_torrent(filepath, os.path.join(metadatapath, sdp, '.torrent')): UpqDB().query("UPDATE file SET torrent=1 WHERE fid=%s" % (fid)) return True
def check(self): if 'file' in self.jobdata: self.jobdata['filename'] = os.path.basename(self.jobdata['file']) result = UpqDB().query( "SELECT * from file where filename LIKE '%s'" % (self.jobdata['filename'])) res = result.first() if res != None: self.jobdata['fid'] = res['fid'] self.msg("File %s already exists: fid: %s" % (self.jobdata['file'], res['fid'])) if not os.access(self.jobdata['file'], os.R_OK): self.msg("Can't access %s" % self.jobdata['file']) return False elif not 'fid' in self.jobdata: self.msg("Either file or fid has to be set!") return False self.enqueue_job() return True
def run(self): """ params: filepath: absoulte path of file filename: filename uid: uid or: fid """ if 'fid' in self.jobdata: # file already exists in db, set filename result = UpqDB().query("SELECT * from file where fid=%s" % (self.jobdata['fid'])) res = result.first() if res == None: self.msg("fid not found in db!") return False self.msg("File already known, Filename: %s Size: %d" % (res['filename'], res['size'])) return True
def update(self, data, mid): """ data is an array with md5 filectime version branch filesize os path """ if data['version'] == "testing": return filename = self.escape(data['path'][data['path'].rfind("/")+1:]) category = "engine_" + data['os'] branch = data['branch'] version = data['version'] if not data['branch'] in ('master'): version = data['version'] + ' ' + data['branch'] url = self.prefix +'/' + data['path'] cid = self.getCID(category) #print "%s %s %s %s" % (filename, version, category, url) try: fid = UpqDB().insert("file", { "filename" : filename, "name": "spring", "version": version, "cid" : cid, "md5" : data['md5'], "timestamp": datetime.datetime.fromtimestamp(data['filectime']), #"timestamp": data['filectime'], "size": data['filesize'], "status": 1 }) except UpqDBIntegrityError, e: try: res = UpqDB().query("SELECT fid from file WHERE version='%s' and cid=%s" % (version, cid)) fid = res.first()[0] UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s"% (data['md5'], fid)) except Exception, e: self.logger.error("Error %s %s %s", version, cid, e) return
def FixPathes(self): from jobs import extract_metadata rows = UpqDB().query( "select filename, fid, name, version, cid, status from file where filename like '/tmp%%'" ) j = extract_metadata.Extract_metadata("extract_metadata", {}) for row in rows: #print(row) srcfile = j.normalizeFilename(row[0], row[2], row[3]) subdir = j.jobcfg['maps-path'] if row[4] == 1 else j.jobcfg[ 'games-path'] status = row[5] #self.movefile(filepath, 1, moveto) prefix = j.getPathByStatus(status) dstfile = os.path.join(prefix, subdir, os.path.basename(srcfile)) filename = os.path.basename(srcfile) UpqDB().query( "UPDATE file SET path='%s', status=%d, filename='%s' WHERE fid=%d" % (subdir, status, filename, row[1]))
def check(self): if 'file' in self.jobdata: if not os.path.exists(self.jobdata['file']): self.logger.error("File %s doesn't exist" % self.jobdata['file']) return False elif 'fid' in self.jobdata: results=UpqDB().query("SELECT filename, path, status FROM file WHERE fid=%s" % (int(self.jobdata['fid']))) res=results.first() if not res: self.logger.error("Fid not found in db") return False prefix=self.getPathByStatus(res['status']) self.jobdata['file']=os.path.join(prefix, res['path'], res['filename'] ) if not os.path.exists(self.jobdata['file']): self.logger.error("filepath from db doesn't exist %s" %(self.jobdata['file'])) return False else: self.logger.error("Either fid or file has to be set") return False self.enqueue_job() return True
def run(self): #username=self.jobcfg['username'] #password=self.jobcfg['password'] proxy = ServerProxy("https://springfiles.com/xmlrpc.php") row = UpqDB().query("SELECT MAX(sid) FROM sf_sync2").first() if row: lastsid = int(row[0]) if row[0] else 0 else: lastsid = 0 self.logger.info("Syncing %d" % (lastsid)) files = proxy.springfiles.getfiles(lastsid) for data in files: sid = int(data["fid"]) row = UpqDB().query("SELECT sid FROM sf_sync2 WHERE sid=%d" % (int(sid))).first() if row: # file found, nothing to do self.logger.debug("File mapping found, continue") continue self.logger.debug("%s %s" % (sid, data)) row = UpqDB().query("SELECT fid FROM file WHERE md5='%s'" % (data["md5"])).first() if row: #UpqDB().insert("sf_sync", {"sid": sid, "fid": row["fid"]}) UpqDB().query( "INSERT INTO sf_sync2 (sid, fid) VALUES (%d, %d) " % (sid, row["fid"])) self.logger.info("Added mapping: %s <-> %s" % (sid, row["fid"])) continue if not self.download_and_add_file("https://springfiles.com/" + data["filepath"]): self.logger.error("Error adding %s" % (data)) else: row = UpqDB().query("SELECT fid FROM file WHERE md5='%s'" % (data["md5"])).first() assert (row) return True
def getmetadata(self, fid): """ returns the metadata, that is required for springfiles in a struct required data: link to images mirror urls file size md5 """ data={} data['mirror']=[] results=UpqDB().query("SELECT CONCAT(m.url_prefix, f.path) as url FROM mirror_file f LEFT JOIN mirror m ON f.mid=m.mid WHERE f.fid=%d" % (fid)) for res in results: data['mirror'].append(res['url']) results=UpqDB().query("SELECT f.filename, f.size, f.timestamp, f.md5, f.name, f.version, c.name as category, f.metadata FROM file f LEFT JOIN categories c ON f.cid=c.cid WHERE f.fid=%d" %(fid)) res=results.first() for value in ["filename", "size", "timestamp", "md5", "name", "version", "category"]: data[value]=res[value] if res['metadata']!=None and len(res['metadata'])>0: data['metadata']=json.loads(res['metadata']) else: data['metadata']=[] return data
def run(self): dled = {} url = self.prefix + '/list.php' filename = "/tmp/sprinvers.json" if not self.DownloadFile(url, filename): self.logger.info("list.php wasn't changed") return True with open(filename, "r") as f: data = json.loads(str(f.read())) res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" % self.prefix) mid = res.first()[0] assert (mid > 0) for row in data: self.update(row, mid) #delete files that wheren't updated this run (means removed from mirror) UpqDB().query( "DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s" % (mid)) return True
def rpc_call_sync(self, proxy, username, password, data): """ make the xml-rpc call """ try: rpcres = proxy.springfiles.sync(username, password, data) for rpc in rpcres: #set description url received from springfiles UpqDB().query( "UPDATE file SET descriptionuri='%s' WHERE fid=%d" % (rpc['url'], rpc['fid'])) except Exception as e: self.msg("xmlrpc springfiles.sync() error: %s" % (e)) return True # fixme self.logger.debug("received from springfiles: %s", rpcres) return True
def deleteOldFiles(self): """ deletes files from ftpmirror when: file is older than 100 days it is known in rapid it is not a stable|test tag (no number at the end in tag) """ results = UpqDB().query( "SELECT m.mid, ftp_url, ftp_user, ftp_pass, ftp_dir, ftp_port, ftp_passive, ftp_ssl \ FROM mirror as m \ WHERE m.status=1") for mirror in results: results2 = UpqDB().query("SELECT f.fid,tag FROM `file` f \ LEFT JOIN tag t ON f.fid=t.fid \ WHERE (t.fid>0) \ AND f.status=1 \ AND timestamp < NOW() - INTERVAL 100 DAY \ GROUP BY f.fid HAVING count(f.fid) = 1") ftp = None for filetodel in results2: if not ftp: try: ftp = self.ftpconnect( mirror['ftp_url'], mirror['ftp_port'], mirror['ftp_user'], mirror['ftp_pass'], mirror['ftp_passive'], mirror['ftp_dir'], mirror['ftp_ssl']) except Exception as e: self.logger.error("Couldn't connect to %s: %s", mirror['ftp_url'], e) break res2 = UpqDB().query( "SELECT * FROM mirror_file WHERE fid = %d AND mid = %d AND status=1" % (filetodel['fid'], mirror['mid'])) curfile = res2.first() if curfile and filetodel['tag'][-1].isdigit(): try: self.logger.error("deleting %s" % (curfile['path'])) ftp.delete(curfile['path']) except: pass UpqDB().query( "UPDATE mirror_file SET status=4 WHERE mfid = %d" % (int(curfile['mfid']))) if ftp: ftp.close()
try: res = UpqDB().query( "SELECT fid from file WHERE version='%s' and cid=%s" % (version, cid)) fid = res.first()[0] UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s" % (data['md5'], fid)) except Exception, e: self.logger.error("Error %s %s %s", version, cid, e) return relpath = self.escape(url[len(self.prefix) + 1:]) try: id = UpqDB().insert( "mirror_file", { "mid": mid, "path": relpath, "status": 1, "fid": fid, "lastcheck": UpqDB().now() }) except UpqDBIntegrityError: res = UpqDB().query( "SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s" % (mid, fid)) id = res.first()[0] UpqDB().query( "UPDATE mirror_file SET lastcheck=NOW() WHERE mfid = %s" % (id)) def run(self): dled = {} url = self.prefix + '/list.php'
def insertData(self, data, filename, hashes): metadata = data.copy() del metadata['Depends'] #remove redundant entries del metadata['sdp'] del metadata['Version'] del metadata['Name'] self.logger.debug(metadata) metadata = json.dumps(metadata) if 'fid' in self.jobdata: # detect already existing files fid = self.jobdata['fid'] else: results = UpqDB().query( "SELECT fid FROM file WHERE sdp='%s' or md5='%s'" % (data['sdp'], hashes['md5'])) res = results.first() if res: fid = res['fid'] else: fid = 0 if fid <= 0: fid = UpqDB().insert( "file", { "name": self.escape(data['Name']), "version": self.escape(data['Version']), "sdp": data['sdp'], "cid": self.getCid(data['Type']), "metadata": metadata, "uid": 0, "path": data["path"], "filename": os.path.basename(filename), "timestamp": UpqDB().now(), #fixme: use file timestamp "size": os.path.getsize(filename), "status": 1, "md5": hashes["md5"], "sha1": hashes["sha1"], "sha256": hashes["sha256"], }) else: UpqDB().query( "UPDATE file SET name='%s', version='%s', sdp='%s', cid=%s, metadata='%s', md5='%s', sha1='%s', sha256='%s', status=1 WHERE fid=%s" % (self.escape(data['Name']), data['Version'], data['sdp'], self.getCid(data['Type']), metadata, hashes["md5"], hashes["sha1"], hashes["sha256"], fid)) # remove already existing depends UpqDB().query("DELETE FROM file_depends WHERE fid = %s" % (fid)) for depend in data['Depends']: res = UpqDB().query( "SELECT fid FROM file WHERE CONCAT(name,' ',version)='%s'" % (depend)) row = res.first() if not row: id = 0 else: id = row['fid'] try: UpqDB().insert("file_depends", { "fid": fid, "depends_string": depend, "depends_fid": id }) self.msg("Added %s '%s' version '%s' to the mirror-system" % (data['Type'], data['Name'], data['Version'])) except UpqDBIntegrityError: pass self.msg("Updated %s '%s' version '%s' sdp '%s' in the mirror-system" % (data['Type'], data['Name'], data['Version'], data['sdp'])) return fid
def update(self, data, mid): """ data is an array with md5 filectime version branch filesize os path """ if data['version'] == "testing": return filename = self.escape(data['path'][data['path'].rfind("/") + 1:]) category = "engine_" + data['os'] branch = data['branch'] version = data['version'] if not data['branch'] in ('master'): version = data['version'] + ' ' + data['branch'] url = self.prefix + '/' + data['path'] cid = self.getCID(category) #print "%s %s %s %s" % (filename, version, category, url) res = UpqDB().query( "SELECT fid from file WHERE version='%s' and cid=%s" % (version, cid)) fid = res.first() if fid: fid = fid[0] assert (fid > 0) UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s" % (data['md5'], fid)) else: fid = UpqDB().insert( "file", { "filename": filename, "name": "spring", "version": version, "cid": cid, "md5": data['md5'], "timestamp": datetime.datetime.fromtimestamp( data['filectime']), #"timestamp": data['filectime'], "size": data['filesize'], "status": 1 }) res = UpqDB().query( "SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s" % (mid, fid)) mfid = res.first() if mfid: mfid = mfid[0] assert (mid > 0) UpqDB().query( "UPDATE mirror_file SET lastcheck=NOW() WHERE mfid = %s" % (mfid)) else: relpath = self.escape(url[len(self.prefix) + 1:]) mfid = UpqDB().insert( "mirror_file", { "mid": mid, "path": relpath, "status": 1, "fid": fid, "lastcheck": UpqDB().now() })
def run(self): fid = int(self.jobdata['fid']) results = UpqDB().query( "SELECT filename, path, size, md5 FROM file where fid=%d AND status=1" % (fid)) if results.rowcount != 1: self.msg("Wrong result count with fid %d" % fid) return False res = results.first() srcfilename = os.path.join(UpqConfig().paths['files'], res['path'], res['filename']) dstfilename = os.path.join(res['path'], res['filename']) filesize = res['size'] md5 = res['md5'] #uploads a fid to all mirrors results = UpqDB().query( "SELECT m.mid, ftp_url, ftp_user, ftp_pass, ftp_dir, ftp_port, ftp_passive, ftp_ssl \ from file f \ left join mirror m on m.status=f.status \ where f.fid=%d and f.status=1 and m.status=1 \ and m.mid not in (select mid from mirror_file where fid=%d and status = 1)" % (fid, fid)) uploadcount = 0 for res in results: try: ftp = self.ftpconnect(res['ftp_url'], res['ftp_port'], res['ftp_user'], res['ftp_pass'], res['ftp_passive'], res['ftp_dir'], res['ftp_ssl']) except Exception as e: self.logger.error("Couldn't connect to the ftp server: %s", e) continue if not os.path.isfile(srcfilename): self.msg("File doesn't exist: " + srcfilename) return False try: f = open(srcfilename, "rb") dstdir = os.path.dirname(dstfilename) try: self.logger.debug("cd into " + dstdir) ftp.cwd(dstdir) except: try: self.logger.debug("mkdir " + dstdir) ftp.mkd(dstdir) self.logger.debug("cwd " + dstdir) ftp.cwd(dstdir) except: self.logger.error( "couldn't cd/mkdir %s, skipping upload " % (dstdir)) continue self.logger.info("uploading %s to %s" % (os.path.basename(dstfilename), (res['ftp_url']))) ftp.storbinary('STOR ' + os.path.basename(dstfilename), f) ftp.quit() f.close() try: #upload succeed, mark in db as uploaded id = UpqDB().insert( "mirror_file", { "mid": res['mid'], "fid": fid, "path": dstfilename, "status": 1 }) except UpqDBIntegrityError: res = UpqDB().query( "SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s" % (res['mid'], fid)) id = res.first() self.logger.info("file already uploaded: mfid=%d" % (id)) UpqDB().query( "UPDATE mirror_file SET status=1 path='%s' WHERE mfid=%s" % (dstfilename, id)) self.logger.debug("inserted into db as %d", id) self.enqueue_newjob("verify_remote_file", {"mfid": id}) except ftplib.all_errors as e: self.logger.error("Ftp-Error (%s) %s failed %s" % ((res['ftp_url'], srcfilename, e))) except Exception as e: self.logger.error("Upload (%s) %s failed %s" % ((res['ftp_url'], srcfilename, e))) return False uploadcount += 1 self.msg("Uploaded to %d mirrors." % (uploadcount)) self.deleteOldFiles() return True
fid = res.first()[0] UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s"% (data['md5'], fid)) except Exception, e: self.logger.error("Error %s %s %s", version, cid, e) return relpath = self.escape(url[len(self.prefix)+1:]) try: id = UpqDB().insert("mirror_file", { "mid" : mid, "path": relpath, "status": 1, "fid": fid, "lastcheck": UpqDB().now() }) except UpqDBIntegrityError: res = UpqDB().query("SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s" % (mid, fid)) id = res.first()[0] UpqDB().query("UPDATE mirror_file SET lastcheck=NOW() WHERE mfid = %s"% (id)) def run(self): dled = {} url = self.prefix + '/list.php' #print self.getlobbyversion() f = my_download().open(url) data = json.loads(str(f.read())) res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" % self.prefix) mid = res.first()[0] for row in data: self.update(row, mid) #delete files that wheren't updated this run (means removed from mirror) UpqDB().query("DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s" %(mid))
def updatefile(self, fid, command): UpqDB().insert("sf_sync", {'fid': fid, 'command': command})
def run(self): if self.jobdata.has_key('fid'): #fid set, add change to db fid = int(self.jobdata['fid']) command = 0 #default to update command if self.jobdata.has_key('command'): command = self.jobdata['command'] if command == "update": command = 0 elif command == "delete": command = 1 else: #unknown command self.logger.error( "Invalid command: %s, only update or delete are valid" % (command)) return False self.updatefile(fid, command) username = self.jobcfg['username'] password = self.jobcfg['password'] proxy = ServerProxy(self.jobcfg['rpcurl']) if self.jobdata.has_key( 'sid'): #sid set, update to springfiles requested sid = int(self.jobdata['sid']) else: #sid not set, request from springfiles try: sid = int(proxy.springfiles.getlastsyncid(username, password)) self.logger.debug("Fetched sid from springfiles: %d" % (sid)) except Exception as e: self.msg("xmlrpc springfiles.getlastsyncid() error: %s" % (e)) return False if sid > 0: results = UpqDB().query( "SELECT fid, command FROM sf_sync WHERE sid>%d ORDER BY sid " % (sid)) #get all changes from db else: results = UpqDB().query( "SELECT fid FROM file WHERE status=1 ORDER BY fid" ) #full sync, get all fids self.logger.debug("springfiles returned %d, doing full sync" % (sid)) lastfid = -1 lastcmd = -1 requests = [] maxsid = UpqDB().query( "SELECT MAX(sid) as sid FROM sf_sync").first()['sid'] for res in results: #fixme: use yield data = {} if sid == 0: #full sync, all commands are update data['command'] = "update" data['metadata'] = self.getmetadata(res['fid']) else: if lastfid == res['fid'] and lastcmd == res[ 'command']: #skip if the same command was already done before continue lastfid = res['fid'] if 'command' in data: lastcmd = data['command'] if res['command'] == 1: # delete data['command'] = "delete" elif res['command'] == 0: # update data['metadata'] = self.getmetadata(res['fid']) data['command'] = "update" else: self.logger.error("unknown command %d for fid %d", res['command'], res['fid']) continue self.logger.debug("adding sid %d to requests" % (res['fid'])) data['fid'] = res['fid'] data['sid'] = maxsid requests.append(data) if len(requests) > int( self.jobcfg['maxrequests'] ): #more then maxrequests queued, flush them self.logger.debug("flushing %d requests" % (len(requests))) if not self.rpc_call_sync(proxy, username, password, requests): return False requests = [] if len(requests) > 0: #update remaining requests return self.rpc_call_sync(proxy, username, password, requests) return True