Example #1
0
    def getmetadata(self, fid):
        """
			returns the metadata, that is required for springfiles in a struct
			required data:
				link to images
				mirror urls
				file size
				md5
		"""
        data = {}
        data['mirror'] = []
        results = UpqDB().query(
            "SELECT CONCAT(m.url_prefix, f.path) as url FROM mirror_file f LEFT JOIN mirror m ON f.mid=m.mid WHERE f.fid=%d"
            % (fid))
        for res in results:
            data['mirror'].append(res['url'])
        results = UpqDB().query(
            "SELECT f.filename, f.size, f.timestamp, f.md5, f.name, f.version, c.name as category, f.metadata FROM file f LEFT JOIN categories c ON f.cid=c.cid WHERE f.fid=%d"
            % (fid))
        res = results.first()
        for value in [
                "filename", "size", "timestamp", "md5", "name", "version",
                "category"
        ]:
            data[value] = res[value]
        if res['metadata'] != None and len(res['metadata']) > 0:
            data['metadata'] = json.loads(res['metadata'])
        else:
            data['metadata'] = []
        return data
Example #2
0
	def run(self):
		"""
			class Hash must be initialized with fileid!
		"""
		fid = int(self.jobdata['fid'])
		results = UpqDB().query("SELECT filename, path, md5, sha1, sha256  FROM file WHERE fid=%d  " % int(fid))
		res=results.first()
		filename = os.path.join(UpqConfig().paths['files'], res['path'], res['filename'])
		if not os.path.exists(filename):
			self.msg("File %s doesn't exist" % (filename))
			return False
		hashes = self.hash(filename)
		if res['md5']!=None and res['md5']!=hashes['md5']:
			self.logger.error("md5 missmatch")
			return False
		if res['sha1']!=None and res['sha1']!=hashes['sha1']:
			self.logger.error("sha1 missmatch")
			return False
		if res['sha256']!=None and res['sha256']!=hashes['sha256']:
			self.logger.error("sha256 missmatch")
			return False
		UpqDB().query("UPDATE file set md5='%s', sha1='%s', sha256='%s' WHERE fid=%d" %
			(hashes['md5'], hashes['sha1'], hashes['sha256'], fid))
		self.msg("md5: %s sha1: %s sha256: %s" % (hashes['md5'], hashes['sha1'], hashes['sha256']))
		return True
Example #3
0
	def normalizeFilename(self, srcfile, fid, name, version):
		""" normalize filename + renames file + updates filename in database """
		name=name.lower()
		if len(version)>0:
			name = str(name[:200] +"-" + version.lower())[:255]
		_, extension = os.path.splitext(srcfile)
		name += extension

		res=""
		for c in name:
			if c in "abcdefghijklmnopqrstuvwxyz-_.01234567890":
				res+=c
			else:
				res+="_"
		dstfile=os.path.join(os.path.dirname(srcfile), res)
		if srcfile!=dstfile:
			results=UpqDB().query("SELECT fid FROM file WHERE BINARY filename='%s' AND status=1" % (res))
			row=results.first()
			if row or os.path.exists(dstfile):
				self.logger.error("Error renaming file: %s to %s already exists, deleting source + using dst!" % (srcfile, dstfile))
				os.remove(srcfile)
			else:
				shutil.move(srcfile, dstfile)
			
			UpqDB().query("UPDATE file SET filename='%s' WHERE fid=%s" %(res, fid))
			self.logger.info("Normalized filename to %s "%(dstfile))
		return dstfile
Example #4
0
 def check(self):
     if 'file' in self.jobdata:
         if not os.path.exists(self.jobdata['file']):
             self.logger.error("File %s doesn't exist" %
                               self.jobdata['file'])
             return False
     elif 'fid' in self.jobdata:
         results = UpqDB().query(
             "SELECT filename, path, status FROM file WHERE fid=%s" %
             (int(self.jobdata['fid'])))
         res = results.first()
         if not res:
             self.logger.error("Fid not found in db")
             return False
         prefix = self.getPathByStatus(res['status'])
         self.jobdata['file'] = os.path.join(prefix, res['path'],
                                             res['filename'])
         if not os.path.exists(self.jobdata['file']):
             self.logger.error("filepath from db doesn't exist %s" %
                               (self.jobdata['file']))
             return False
     else:
         self.logger.error("Either fid or file has to be set")
         return False
     self.enqueue_job()
     return True
Example #5
0
    def run(self):
        """
			class Hash must be initialized with fileid!
		"""
        fid = int(self.jobdata['fid'])
        results = UpqDB().query(
            "SELECT filename, path, md5, sha1, sha256  FROM file WHERE fid=%d  "
            % int(fid))
        res = results.first()
        filename = os.path.join(UpqConfig().paths['files'], res['path'],
                                res['filename'])
        if not os.path.exists(filename):
            self.msg("File %s doesn't exist" % (filename))
            return False
        hashes = self.hash(filename)
        if res['md5'] != None and res['md5'] != hashes['md5']:
            self.logger.error("md5 missmatch")
            return False
        if res['sha1'] != None and res['sha1'] != hashes['sha1']:
            self.logger.error("sha1 missmatch")
            return False
        if res['sha256'] != None and res['sha256'] != hashes['sha256']:
            self.logger.error("sha256 missmatch")
            return False
        UpqDB().query(
            "UPDATE file set md5='%s', sha1='%s', sha256='%s' WHERE fid=%d" %
            (hashes['md5'], hashes['sha1'], hashes['sha256'], fid))
        self.msg("md5: %s sha1: %s sha256: %s" %
                 (hashes['md5'], hashes['sha1'], hashes['sha256']))
        return True
Example #6
0
	def getCid(self, name):
		result=UpqDB().query("SELECT cid from categories WHERE name='%s'" % name)
		res=result.first()
		if res:
			cid=res['cid']
		else:
			cid=UpqDB().insert("categories", {"name": name})
		return cid
Example #7
0
 def getCid(self, name):
     result = UpqDB().query("SELECT cid from categories WHERE name='%s'" %
                            name)
     res = result.first()
     if res:
         cid = res['cid']
     else:
         cid = UpqDB().insert("categories", {"name": name})
     return cid
Example #8
0
	def getCID(self, category):
		if category in self.cats:
			return self.cats[category]
		res = UpqDB().query("SELECT cid from categories WHERE name='%s'" % (category))
		try:
			self.cats[category]=res.first()[0] # cache result
		except:
			self.logger.error("Invalid category: %s" % category)
		return self.cats[category]
Example #9
0
	def check(self):
		results = UpqDB().query("SELECT fid, filename, path FROM file WHERE fid=%d " % int(self.jobdata['fid']))
		res=results.first()
		filename = os.path.join(UpqConfig().paths['files'], res['path'], res['filename'])
		if not os.path.exists(filename):
			self.logger.error("file %d doesn't exist: %s"%(res['fid'], filename))
			return False
		self.enqueue_job()
		return True
Example #10
0
	def insertData(self, data, filename):
		metadata=data.copy()
		del metadata['Depends'] #remove redundant entries
		del metadata['sdp']
		del metadata['Version']
		del metadata['Name']
		metadata=json.dumps(metadata)
		if 'fid' in self.jobdata: # detect already existing files
			fid=self.jobdata['fid']
		else:
			results=UpqDB().query("SELECT fid FROM file WHERE sdp='%s'"% (data['sdp']))
			res=results.first()
			if res:
				fid=res['fid']
			else:
				fid=0

		if fid<=0:
			fid=UpqDB().insert("file", {
				"name": self.escape(data['Name']),
				"version": self.escape(data['Version']),
				"sdp": data['sdp'],
				"cid": self.getCid(data['Type']),
				"metadata": metadata,
				"uid": 0,
				"path": "",
				"filename": filename,
				"timestamp": UpqDB().now(), #fixme: use file timestamp
				"size": os.path.getsize(filename),
				"status": 1,
				})
		else:
			UpqDB().query("UPDATE file SET name='%s', version='%s', sdp='%s', cid=%s, metadata='%s' WHERE fid=%s" %(
				self.escape(data['Name']),
				data['Version'],
				data['sdp'],
				self.getCid(data['Type']),
				metadata,
				fid
				))
		# remove already existing depends
		UpqDB().query("DELETE FROM file_depends WHERE fid = %s" % (fid) )
		for depend in data['Depends']:
			res=UpqDB().query("SELECT fid FROM file WHERE CONCAT(name,' ',version)='%s'" % (depend))
			row=res.first()
			if not row:
				id=0
			else:
				id=row['fid']
			try:
				UpqDB().insert("file_depends", {"fid":fid, "depends_string": depend, "depends_fid": id})
				self.msg("Added %s '%s' version '%s' to the mirror-system" % (data['Type'], data['Name'], data['Version']))
			except UpqDBIntegrityError:
				pass
		self.msg("Updated %s '%s' version '%s' sdp '%s' in the mirror-system" % (data['Type'], data['Name'], data['Version'], data['sdp']))
		return fid
Example #11
0
 def getCID(self, category):
     if category in self.cats:
         return self.cats[category]
     res = UpqDB().query("SELECT cid from categories WHERE name='%s'" %
                         (category))
     try:
         self.cats[category] = res.first()[0]  # cache result
     except:
         self.logger.error("Invalid category: %s" % category)
     return self.cats[category]
Example #12
0
    def check(self):
        if not "fid" in self.jobdata:
            self.msg("fid not specified")
            return False

        results = UpqDB().query("SELECT * FROM file WHERE fid=%s and status=1" % (int(self.jobdata["fid"])))
        res = results.first()
        if res == None:
            self.msg("fid not found")
            return False
        id = self.enqueue_job()
        return True
Example #13
0
	def check(self):
		if not 'fid' in self.jobdata:
			self.msg("fid not specified")
			return False

		results=UpqDB().query("SELECT * FROM file WHERE fid=%s and status=1" % (int(self.jobdata['fid'])))
		res=results.first()
		if res == None:
			self.msg("fid not found")
			return False
		id=self.enqueue_job()
		return True
Example #14
0
 def task_done(self, job):
     # update job state in DB
     if job.result:
         result = 0
     else:
         result = 1
     if len(job.msgstr) > 255:  #limit string length
         job.msgstr = job.msgstr[:255]
     query = "UPDATE upqueue SET status=0, end_time=NOW(), result_msg='%s' WHERE jobid = %s" % (
         UpqDB().escape(job.msgstr), job.jobid)
     UpqDB().query(query)
     super(DBQueue, self).task_done()
Example #15
0
 def check(self):
     results = UpqDB().query(
         "SELECT fid, filename, path FROM file WHERE fid=%d " %
         int(self.jobdata['fid']))
     res = results.first()
     filename = os.path.join(UpqConfig().paths['files'], res['path'],
                             res['filename'])
     if not os.path.exists(filename):
         self.logger.error("file %d doesn't exist: %s" %
                           (res['fid'], filename))
         return False
     self.enqueue_job()
     return True
Example #16
0
    def run(self):
        results = UpqDB().query("SELECT count(mid) from mirror")
        count = res.first()[0]
        results = UpqDB().query(
            "SELECT f.mfid FROM mirror_file f LEFT JOIN mirror m ON f.mid=m.mid \
			WHERE m.status=1 \
			AND ( f.lastcheck < DATE_SUB(NOW(), INTERVAL 1 MONTH) \
			OR f.lastcheck is null ) \
			ORDER BY f.lastcheck ASC \
			LIMIT 0,30")
        for res in results:
            self.enqueue_newjob("verify_remote_file", {"mfid": res["mfid"]})
            sleep(10)
        return True
Example #17
0
 def put(self, job, block=True, timeout=None):
     if job.jobid == -1:
         # add job to DB in state "new"
         jobdata = json.dumps(job.jobdata, -1)
         ret = UpqDB().insert(
             "upqueue", {
                 'jobname': job.jobname,
                 'status': 2,
                 'jobdata': jobdata,
                 'ctime': UpqDB().now()
             })
         job.jobid = ret  #set jobid
     super(DBQueue, self).put(job, block, timeout)
     return job.jobid
Example #18
0
	def run(self):
		dled = {}
		url = self.prefix + '/list.php'
		#print self.getlobbyversion()
		f = my_download().open(url)
		data = json.loads(str(f.read()))
		res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" % self.prefix)
		mid = res.first()[0]
		for row in data:
			self.update(row, mid)
		#delete files that wheren't updated this run (means removed from mirror)
		UpqDB().query("DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s" %(mid))
		urllib.urlcleanup()
		return True
Example #19
0
    def update(self, data, mid):
        """
			data is an array with
				md5
				filectime
				version
				branch
				filesize
				os
				path
		"""
        if data['version'] == "testing":
            return
        filename = self.escape(data['path'][data['path'].rfind("/") + 1:])
        category = "engine_" + data['os']
        branch = data['branch']
        version = data['version']
        if not data['branch'] in ('master'):
            version = data['version'] + ' ' + data['branch']
        url = self.prefix + '/' + data['path']
        cid = self.getCID(category)
        #print "%s %s %s %s" % (filename, version, category, url)
        try:
            fid = UpqDB().insert(
                "file",
                {
                    "filename": filename,
                    "name": "spring",
                    "version": version,
                    "cid": cid,
                    "md5": data['md5'],
                    "timestamp": datetime.datetime.fromtimestamp(
                        data['filectime']),
                    #"timestamp": data['filectime'],
                    "size": data['filesize'],
                    "status": 1
                })
        except UpqDBIntegrityError, e:
            try:
                res = UpqDB().query(
                    "SELECT fid from file WHERE version='%s' and cid=%s" %
                    (version, cid))
                fid = res.first()[0]
                UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s" %
                              (data['md5'], fid))
            except Exception, e:
                self.logger.error("Error %s %s %s", version, cid, e)
                return
Example #20
0
	def run(self):
		fid=int(self.jobdata['fid'])
		results=UpqDB().query("SELECT filename, path, sdp FROM file WHERE fid=%d AND status=1" % fid)
		res=results.first()
		#filename of the archive to be scanned
		filename=res['filename'] # filename only (no path info)
		absfilename=os.path.join(UpqConfig().paths['files'], res['path'], res['filename']) # absolute filename
		torrent=os.path.join(UpqConfig().paths['metadata'], res['sdp']+ ".torrent" )
		if not os.path.exists(absfilename):
			self.msg("File doesn't exist: %s" %(absfilename))
			return False

		res=self.create_torrent(absfilename, torrent)
		if res:
			UpqDB().query("UPDATE file SET torrent=1 WHERE fid=%s" %(fid))
		return res
Example #21
0
    def run(self):
        fid = int(self.jobdata["fid"])
        results = UpqDB().query("SELECT filename, path, sdp FROM file WHERE fid=%d AND status=1" % fid)
        res = results.first()
        # filename of the archive to be scanned
        filename = res["filename"]  # filename only (no path info)
        absfilename = os.path.join(UpqConfig().paths["files"], res["path"], res["filename"])  # absolute filename
        torrent = os.path.join(UpqConfig().paths["metadata"], res["sdp"] + ".torrent")
        if not os.path.exists(absfilename):
            self.msg("File doesn't exist: %s" % (absfilename))
            return False

        res = self.create_torrent(absfilename, torrent)
        if res:
            UpqDB().query("UPDATE file SET torrent=1 WHERE fid=%s" % (fid))
        return res
Example #22
0
	def check(self):
		if 'file' in self.jobdata :
			self.jobdata['filename']=os.path.basename(self.jobdata['file'])
			result=UpqDB().query("SELECT * from file where filename LIKE '%s'" % (self.jobdata['filename']))
			res=result.first()
			if res!=None:
				self.jobdata['fid']=res['fid']
				self.msg("File %s already exists: fid: %s"%(self.jobdata['file'], res['fid']))
			if not os.access(self.jobdata['file'], os.R_OK):
				self.msg("Can't access %s"% self.jobdata['file'])
				return False
		elif not 'fid' in self.jobdata:
			self.msg("Either file or fid has to be set!")
			return False
		self.enqueue_job()
		return True
Example #23
0
 def get(self, block=True, timeout=None):
     job = super(DBQueue, self).get(block, timeout)
     # update job state in DB
     query = "UPDATE upqueue SET status=1, start_time=NOW() WHERE jobid = %d" % (
         job.jobid)
     UpqDB().query(query)
     return job
Example #24
0
 def run(self):
     dled = {}
     url = self.prefix + '/list.php'
     #print self.getlobbyversion()
     f = my_download().open(url)
     data = json.loads(str(f.read()))
     res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" %
                         self.prefix)
     mid = res.first()[0]
     for row in data:
         self.update(row, mid)
     #delete files that wheren't updated this run (means removed from mirror)
     UpqDB().query(
         "DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s"
         % (mid))
     urllib.urlcleanup()
     return True
Example #25
0
	def run(self):
		"""
			params:
				filepath: absoulte path of file
				filename: filename
				uid: uid
			or:
				fid
		"""
		if 'fid' in self.jobdata: # file already exists in db, set filename
			result=UpqDB().query("SELECT * from file where fid=%s"% (self.jobdata['fid']))
			res=result.first()
			if res==None:
				self.msg("fid not found in db!")
				return False
			self.msg("File already known, Filename: %s Size: %d" % (res['filename'], res['size']))
		return True
Example #26
0
	def UpdateSDP(self, sdp):
		"""
		values = {
				"tag"=sdp[0],
				"md5"=sdp[1],
				"depends"=sdp[2],
				"name"=sdp[3],
		}
		"""
		#check if file is already known
		res=UpqDB().query("SELECT f.fid FROM file f \
			LEFT JOIN tag t ON t.fid=f.fid \
			WHERE sdp='%s'" % (sdp[1]))
		row=res.first()
		if row: #file is already known
			#delete tag from existing files
			UpqDB().query("DELETE FROM tag WHERE tag='%s'" % (sdp[0]))
			#insert updated tag
			UpqDB().query("INSERT INTO tag (fid, tag) VALUES (%s, '%s')" % (row['fid'], sdp[0]))
			#self.logger.info("updated %s %s %s %s",sdp[3],sdp[2],sdp[1],sdp[0])
			return
		if not sdp[3]: #without a name, we can't do anything!
			return
		cid = self.getCid("game")
		try:
			fid = UpqDB().insert("file", {
				"filename" : sdp[3] + " (not available as sdz)",
				"name": sdp[3],
				"cid" : cid,
				"md5" : sdp[1],
				"sdp" : sdp[1],
				"size" : 0,
				"status" : 4, # special status for files that can only be downloaded via rapid
				"uid" : 0,
				"path" : "",
				})
			UpqDB().query("INSERT INTO tag (fid, tag) VALUES (%s, '%s')" % (fid, sdp[0]))
			#self.logger.info("inserted %s %s %s %s",sdp[3],sdp[2],sdp[1],sdp[0])
		except Exception as e:
			self.logger.error(str(e))
			self.logger.error("Error from sdp: %s %s %s %s", sdp[3], sdp[2],sdp[1],sdp[0])
			res=UpqDB().query("SELECT * FROM file f WHERE name='%s'" % sdp[3])
			if res:
				row=res.first()
				self.logger.error("a file with this name already exists, fid=%s, sdp=%s" % (row['fid'], row['sdp']))
Example #27
0
    def ExtractMetadata(self, usync, archiveh, filename, filepath,
                        metadatapath, hashes):
        filelist = self.getFileList(usync, archiveh)
        sdp = self.getSDPName(usync, archiveh)
        idx = self.getMapIdx(usync, filename)
        if idx >= 0:  #file is map
            archivepath = usync.GetArchivePath(
                filename.encode()).decode() + filename
            springname = usync.GetMapName(idx).decode()
            data = self.getMapData(usync, filename, idx, archiveh, springname)
            data['mapimages'] = self.dumpmap(usync, springname, metadatapath,
                                             filename, idx)
            data['path'] = self.jobcfg['maps-path']
        else:  # file is a game
            idx = self.getGameIdx(usync, filename)
            if idx < 0:
                self.logger.error("Invalid file detected: %s %s %s" %
                                  (filename, usync.GetNextError(), idx))
                return False
            self.logger.debug("Extracting data from " + filename)
            archivepath = usync.GetArchivePath(filename) + filename
            gamearchivecount = usync.GetPrimaryModArchiveCount(
                idx)  # initialization for GetPrimaryModArchiveList()
            data = self.getGameData(usync, idx, gamearchivecount, archivepath,
                                    archiveh)
            data['path'] = self.jobcfg['games-path']
        if (sdp == "") or (data['Name'] == ""
                           ):  #mark as broken because sdp / name is missing
            self.logger.error("Couldn't get name / filename")
            return False
        data['splash'] = self.createSplashImages(usync, archiveh, filelist)
        _, extension = os.path.splitext(filename)
        moveto = os.path.join(
            self.getPathByStatus(1), data['path'],
            self.GetNormalizedFilename(data['Name'], data['Version'],
                                       extension))
        self.jobdata['file'] = moveto

        assert (moveto != filepath)
        assert (len(moveto) > 0)

        if not self.movefile(filepath, moveto):
            self.logger.error("Couldn't move file %s -> %s" %
                              (filepath, moveto))
            return False
        try:
            data['sdp'] = sdp
            self.jobdata['fid'] = self.insertData(data, moveto, hashes)
        except UpqDBIntegrityError:
            self.logger.error("Duplicate file detected: %s %s %s" %
                              (filename, data['Name'], data['Version']))
            return False

        if self.create_torrent(filepath,
                               os.path.join(metadatapath, sdp, '.torrent')):
            UpqDB().query("UPDATE file SET torrent=1 WHERE fid=%s" % (fid))
        return True
Example #28
0
 def check(self):
     if 'file' in self.jobdata:
         self.jobdata['filename'] = os.path.basename(self.jobdata['file'])
         result = UpqDB().query(
             "SELECT * from file where filename LIKE '%s'" %
             (self.jobdata['filename']))
         res = result.first()
         if res != None:
             self.jobdata['fid'] = res['fid']
             self.msg("File %s already exists: fid: %s" %
                      (self.jobdata['file'], res['fid']))
         if not os.access(self.jobdata['file'], os.R_OK):
             self.msg("Can't access %s" % self.jobdata['file'])
             return False
     elif not 'fid' in self.jobdata:
         self.msg("Either file or fid has to be set!")
         return False
     self.enqueue_job()
     return True
Example #29
0
    def run(self):
        """
			params:
				filepath: absoulte path of file
				filename: filename
				uid: uid
			or:
				fid
		"""
        if 'fid' in self.jobdata:  # file already exists in db, set filename
            result = UpqDB().query("SELECT * from file where fid=%s" %
                                   (self.jobdata['fid']))
            res = result.first()
            if res == None:
                self.msg("fid not found in db!")
                return False
            self.msg("File already known, Filename: %s Size: %d" %
                     (res['filename'], res['size']))
        return True
Example #30
0
	def update(self, data, mid):
		"""
			data is an array with
				md5
				filectime
				version
				branch
				filesize
				os
				path
		"""
		if data['version'] == "testing":
			return
		filename = self.escape(data['path'][data['path'].rfind("/")+1:])
		category = "engine_" + data['os']
		branch = data['branch']
		version = data['version']
		if not data['branch'] in ('master'):
			version = data['version'] + ' ' + data['branch']
		url = self.prefix +'/' + data['path']
		cid = self.getCID(category)
		#print "%s %s %s %s" % (filename, version, category, url)
		try:
			fid = UpqDB().insert("file", {
				"filename" : filename,
				"name": "spring",
				"version": version,
				"cid" : cid,
				"md5" : data['md5'],
				"timestamp": datetime.datetime.fromtimestamp(data['filectime']),
				#"timestamp": data['filectime'],
				"size": data['filesize'],
				"status": 1 })
		except UpqDBIntegrityError, e:
			try:
				res = UpqDB().query("SELECT fid from file WHERE version='%s' and cid=%s" % (version, cid))
				fid = res.first()[0]
				UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s"%  (data['md5'], fid))
			except Exception, e:
				self.logger.error("Error %s %s %s", version, cid, e)
				return
Example #31
0
    def FixPathes(self):
        from jobs import extract_metadata
        rows = UpqDB().query(
            "select filename, fid, name, version, cid, status from file where filename like '/tmp%%'"
        )
        j = extract_metadata.Extract_metadata("extract_metadata", {})
        for row in rows:
            #print(row)
            srcfile = j.normalizeFilename(row[0], row[2], row[3])
            subdir = j.jobcfg['maps-path'] if row[4] == 1 else j.jobcfg[
                'games-path']

            status = row[5]
            #self.movefile(filepath, 1, moveto)

            prefix = j.getPathByStatus(status)
            dstfile = os.path.join(prefix, subdir, os.path.basename(srcfile))
            filename = os.path.basename(srcfile)
            UpqDB().query(
                "UPDATE file SET path='%s', status=%d, filename='%s' WHERE fid=%d"
                % (subdir, status, filename, row[1]))
Example #32
0
	def check(self):
		if 'file' in self.jobdata:
			if not os.path.exists(self.jobdata['file']):
				self.logger.error("File %s doesn't exist" % self.jobdata['file'])
				return False
		elif 'fid' in self.jobdata:
			results=UpqDB().query("SELECT filename, path, status FROM file WHERE fid=%s" % (int(self.jobdata['fid'])))
			res=results.first()
			if not res:
				self.logger.error("Fid not found in db")
				return False
			prefix=self.getPathByStatus(res['status'])
			self.jobdata['file']=os.path.join(prefix, res['path'], res['filename'] )
			if not os.path.exists(self.jobdata['file']):
				self.logger.error("filepath from db doesn't exist %s" %(self.jobdata['file']))
				return False
		else:
			self.logger.error("Either fid or file has to be set")
			return False
		self.enqueue_job()
		return True
Example #33
0
    def run(self):
        #username=self.jobcfg['username']
        #password=self.jobcfg['password']
        proxy = ServerProxy("https://springfiles.com/xmlrpc.php")

        row = UpqDB().query("SELECT MAX(sid) FROM sf_sync2").first()
        if row:
            lastsid = int(row[0]) if row[0] else 0
        else:
            lastsid = 0
        self.logger.info("Syncing %d" % (lastsid))
        files = proxy.springfiles.getfiles(lastsid)
        for data in files:
            sid = int(data["fid"])
            row = UpqDB().query("SELECT sid FROM sf_sync2 WHERE sid=%d" %
                                (int(sid))).first()
            if row:  # file found, nothing to do
                self.logger.debug("File mapping found, continue")
                continue

            self.logger.debug("%s %s" % (sid, data))
            row = UpqDB().query("SELECT fid FROM file WHERE md5='%s'" %
                                (data["md5"])).first()
            if row:
                #UpqDB().insert("sf_sync", {"sid": sid, "fid": row["fid"]})
                UpqDB().query(
                    "INSERT INTO sf_sync2 (sid, fid) VALUES (%d, %d) " %
                    (sid, row["fid"]))
                self.logger.info("Added mapping: %s <-> %s" %
                                 (sid, row["fid"]))
                continue
            if not self.download_and_add_file("https://springfiles.com/" +
                                              data["filepath"]):
                self.logger.error("Error adding %s" % (data))
            else:
                row = UpqDB().query("SELECT fid FROM file WHERE md5='%s'" %
                                    (data["md5"])).first()
                assert (row)
        return True
Example #34
0
	def getmetadata(self, fid):
		"""
			returns the metadata, that is required for springfiles in a struct
			required data:
				link to images
				mirror urls
				file size
				md5
		"""
		data={}
		data['mirror']=[]
		results=UpqDB().query("SELECT CONCAT(m.url_prefix, f.path) as url FROM mirror_file f LEFT JOIN mirror m ON f.mid=m.mid WHERE f.fid=%d" % (fid))
		for res in results:
			data['mirror'].append(res['url'])
		results=UpqDB().query("SELECT f.filename, f.size, f.timestamp, f.md5, f.name, f.version, c.name as category, f.metadata FROM file f LEFT JOIN categories c ON f.cid=c.cid WHERE f.fid=%d" %(fid))
		res=results.first()
		for value in ["filename", "size", "timestamp", "md5", "name", "version", "category"]:
			data[value]=res[value]
		if res['metadata']!=None and len(res['metadata'])>0:
			data['metadata']=json.loads(res['metadata'])
		else:
			data['metadata']=[]
		return data
Example #35
0
    def run(self):
        dled = {}
        url = self.prefix + '/list.php'

        filename = "/tmp/sprinvers.json"

        if not self.DownloadFile(url, filename):
            self.logger.info("list.php wasn't changed")
            return True

        with open(filename, "r") as f:
            data = json.loads(str(f.read()))
        res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" %
                            self.prefix)
        mid = res.first()[0]
        assert (mid > 0)
        for row in data:
            self.update(row, mid)
        #delete files that wheren't updated this run (means removed from mirror)
        UpqDB().query(
            "DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s"
            % (mid))
        return True
Example #36
0
    def rpc_call_sync(self, proxy, username, password, data):
        """
			make the xml-rpc call
		"""
        try:
            rpcres = proxy.springfiles.sync(username, password, data)
            for rpc in rpcres:  #set description url received from springfiles
                UpqDB().query(
                    "UPDATE file SET descriptionuri='%s' WHERE fid=%d" %
                    (rpc['url'], rpc['fid']))
        except Exception as e:
            self.msg("xmlrpc springfiles.sync() error: %s" % (e))
            return True  # fixme
        self.logger.debug("received from springfiles: %s", rpcres)
        return True
Example #37
0
    def deleteOldFiles(self):
        """
			deletes files from ftpmirror when:
				file is older than 100 days
				it is known in rapid
				it is not a stable|test tag (no number at the end in tag)
		"""
        results = UpqDB().query(
            "SELECT m.mid, ftp_url, ftp_user, ftp_pass, ftp_dir, ftp_port, ftp_passive, ftp_ssl \
FROM mirror as m \
WHERE m.status=1")
        for mirror in results:
            results2 = UpqDB().query("SELECT f.fid,tag FROM `file` f \
LEFT JOIN tag t ON f.fid=t.fid \
WHERE (t.fid>0) \
AND f.status=1 \
AND timestamp < NOW() - INTERVAL 100 DAY \
GROUP BY f.fid HAVING count(f.fid) = 1")
            ftp = None
            for filetodel in results2:
                if not ftp:
                    try:
                        ftp = self.ftpconnect(
                            mirror['ftp_url'], mirror['ftp_port'],
                            mirror['ftp_user'], mirror['ftp_pass'],
                            mirror['ftp_passive'], mirror['ftp_dir'],
                            mirror['ftp_ssl'])
                    except Exception as e:
                        self.logger.error("Couldn't connect to %s: %s",
                                          mirror['ftp_url'], e)
                        break
                res2 = UpqDB().query(
                    "SELECT * FROM mirror_file WHERE fid = %d AND mid = %d AND status=1"
                    % (filetodel['fid'], mirror['mid']))
                curfile = res2.first()
                if curfile and filetodel['tag'][-1].isdigit():
                    try:
                        self.logger.error("deleting %s" % (curfile['path']))
                        ftp.delete(curfile['path'])
                    except:
                        pass
                    UpqDB().query(
                        "UPDATE mirror_file SET status=4 WHERE mfid = %d" %
                        (int(curfile['mfid'])))
            if ftp:
                ftp.close()
Example #38
0
            try:
                res = UpqDB().query(
                    "SELECT fid from file WHERE version='%s' and cid=%s" %
                    (version, cid))
                fid = res.first()[0]
                UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s" %
                              (data['md5'], fid))
            except Exception, e:
                self.logger.error("Error %s %s %s", version, cid, e)
                return
        relpath = self.escape(url[len(self.prefix) + 1:])
        try:
            id = UpqDB().insert(
                "mirror_file", {
                    "mid": mid,
                    "path": relpath,
                    "status": 1,
                    "fid": fid,
                    "lastcheck": UpqDB().now()
                })
        except UpqDBIntegrityError:
            res = UpqDB().query(
                "SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s" %
                (mid, fid))
            id = res.first()[0]
            UpqDB().query(
                "UPDATE mirror_file SET lastcheck=NOW() WHERE mfid = %s" %
                (id))

    def run(self):
        dled = {}
        url = self.prefix + '/list.php'
Example #39
0
    def insertData(self, data, filename, hashes):
        metadata = data.copy()
        del metadata['Depends']  #remove redundant entries
        del metadata['sdp']
        del metadata['Version']
        del metadata['Name']
        self.logger.debug(metadata)
        metadata = json.dumps(metadata)
        if 'fid' in self.jobdata:  # detect already existing files
            fid = self.jobdata['fid']
        else:
            results = UpqDB().query(
                "SELECT fid FROM file WHERE sdp='%s' or md5='%s'" %
                (data['sdp'], hashes['md5']))
            res = results.first()
            if res:
                fid = res['fid']
            else:
                fid = 0

        if fid <= 0:
            fid = UpqDB().insert(
                "file",
                {
                    "name": self.escape(data['Name']),
                    "version": self.escape(data['Version']),
                    "sdp": data['sdp'],
                    "cid": self.getCid(data['Type']),
                    "metadata": metadata,
                    "uid": 0,
                    "path": data["path"],
                    "filename": os.path.basename(filename),
                    "timestamp": UpqDB().now(),  #fixme: use file timestamp
                    "size": os.path.getsize(filename),
                    "status": 1,
                    "md5": hashes["md5"],
                    "sha1": hashes["sha1"],
                    "sha256": hashes["sha256"],
                })
        else:
            UpqDB().query(
                "UPDATE file SET name='%s', version='%s', sdp='%s', cid=%s, metadata='%s', md5='%s', sha1='%s', sha256='%s', status=1 WHERE fid=%s"
                % (self.escape(data['Name']), data['Version'], data['sdp'],
                   self.getCid(data['Type']), metadata, hashes["md5"],
                   hashes["sha1"], hashes["sha256"], fid))
        # remove already existing depends
        UpqDB().query("DELETE FROM file_depends WHERE fid = %s" % (fid))
        for depend in data['Depends']:
            res = UpqDB().query(
                "SELECT fid FROM file WHERE CONCAT(name,' ',version)='%s'" %
                (depend))
            row = res.first()
            if not row:
                id = 0
            else:
                id = row['fid']
            try:
                UpqDB().insert("file_depends", {
                    "fid": fid,
                    "depends_string": depend,
                    "depends_fid": id
                })
                self.msg("Added %s '%s' version '%s' to the mirror-system" %
                         (data['Type'], data['Name'], data['Version']))
            except UpqDBIntegrityError:
                pass
        self.msg("Updated %s '%s' version '%s' sdp '%s' in the mirror-system" %
                 (data['Type'], data['Name'], data['Version'], data['sdp']))
        return fid
Example #40
0
    def update(self, data, mid):
        """
			data is an array with
				md5
				filectime
				version
				branch
				filesize
				os
				path
		"""
        if data['version'] == "testing":
            return
        filename = self.escape(data['path'][data['path'].rfind("/") + 1:])
        category = "engine_" + data['os']
        branch = data['branch']
        version = data['version']
        if not data['branch'] in ('master'):
            version = data['version'] + ' ' + data['branch']
        url = self.prefix + '/' + data['path']
        cid = self.getCID(category)
        #print "%s %s %s %s" % (filename, version, category, url)
        res = UpqDB().query(
            "SELECT fid from file WHERE version='%s' and cid=%s" %
            (version, cid))
        fid = res.first()
        if fid:
            fid = fid[0]
            assert (fid > 0)
            UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s" %
                          (data['md5'], fid))
        else:
            fid = UpqDB().insert(
                "file",
                {
                    "filename": filename,
                    "name": "spring",
                    "version": version,
                    "cid": cid,
                    "md5": data['md5'],
                    "timestamp": datetime.datetime.fromtimestamp(
                        data['filectime']),
                    #"timestamp": data['filectime'],
                    "size": data['filesize'],
                    "status": 1
                })

        res = UpqDB().query(
            "SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s" %
            (mid, fid))
        mfid = res.first()
        if mfid:
            mfid = mfid[0]
            assert (mid > 0)
            UpqDB().query(
                "UPDATE mirror_file SET lastcheck=NOW() WHERE mfid = %s" %
                (mfid))
        else:
            relpath = self.escape(url[len(self.prefix) + 1:])
            mfid = UpqDB().insert(
                "mirror_file", {
                    "mid": mid,
                    "path": relpath,
                    "status": 1,
                    "fid": fid,
                    "lastcheck": UpqDB().now()
                })
Example #41
0
    def run(self):
        fid = int(self.jobdata['fid'])
        results = UpqDB().query(
            "SELECT filename, path, size, md5 FROM file where fid=%d AND status=1"
            % (fid))
        if results.rowcount != 1:
            self.msg("Wrong result count with fid %d" % fid)
            return False
        res = results.first()
        srcfilename = os.path.join(UpqConfig().paths['files'], res['path'],
                                   res['filename'])
        dstfilename = os.path.join(res['path'], res['filename'])
        filesize = res['size']
        md5 = res['md5']
        #uploads a fid to all mirrors
        results = UpqDB().query(
            "SELECT m.mid, ftp_url, ftp_user, ftp_pass, ftp_dir, ftp_port, ftp_passive, ftp_ssl \
						from file f  \
						left join mirror m on m.status=f.status \
						where f.fid=%d and f.status=1 and m.status=1 \
						and m.mid not in (select mid from mirror_file where fid=%d and status = 1)"
            % (fid, fid))
        uploadcount = 0
        for res in results:
            try:
                ftp = self.ftpconnect(res['ftp_url'], res['ftp_port'],
                                      res['ftp_user'], res['ftp_pass'],
                                      res['ftp_passive'], res['ftp_dir'],
                                      res['ftp_ssl'])
            except Exception as e:
                self.logger.error("Couldn't connect to the ftp server: %s", e)
                continue
            if not os.path.isfile(srcfilename):
                self.msg("File doesn't exist: " + srcfilename)
                return False
            try:
                f = open(srcfilename, "rb")

                dstdir = os.path.dirname(dstfilename)
                try:
                    self.logger.debug("cd into " + dstdir)
                    ftp.cwd(dstdir)
                except:
                    try:
                        self.logger.debug("mkdir " + dstdir)
                        ftp.mkd(dstdir)
                        self.logger.debug("cwd " + dstdir)
                        ftp.cwd(dstdir)
                    except:
                        self.logger.error(
                            "couldn't cd/mkdir %s, skipping upload " %
                            (dstdir))
                        continue
                self.logger.info("uploading %s to %s" %
                                 (os.path.basename(dstfilename),
                                  (res['ftp_url'])))
                ftp.storbinary('STOR ' + os.path.basename(dstfilename), f)

                ftp.quit()
                f.close()
                try:  #upload succeed, mark in db as uploaded
                    id = UpqDB().insert(
                        "mirror_file", {
                            "mid": res['mid'],
                            "fid": fid,
                            "path": dstfilename,
                            "status": 1
                        })
                except UpqDBIntegrityError:
                    res = UpqDB().query(
                        "SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s"
                        % (res['mid'], fid))
                    id = res.first()
                    self.logger.info("file already uploaded: mfid=%d" % (id))
                    UpqDB().query(
                        "UPDATE mirror_file SET status=1 path='%s' WHERE mfid=%s"
                        % (dstfilename, id))
                self.logger.debug("inserted into db as %d", id)
                self.enqueue_newjob("verify_remote_file", {"mfid": id})
            except ftplib.all_errors as e:
                self.logger.error("Ftp-Error (%s) %s failed %s" %
                                  ((res['ftp_url'], srcfilename, e)))
            except Exception as e:
                self.logger.error("Upload (%s) %s failed %s" %
                                  ((res['ftp_url'], srcfilename, e)))
                return False
            uploadcount += 1
        self.msg("Uploaded to %d mirrors." % (uploadcount))
        self.deleteOldFiles()
        return True
Example #42
0
				fid = res.first()[0]
				UpqDB().query("UPDATE file set md5='%s' WHERE fid=%s"%  (data['md5'], fid))
			except Exception, e:
				self.logger.error("Error %s %s %s", version, cid, e)
				return
		relpath = self.escape(url[len(self.prefix)+1:])
		try:
			id = UpqDB().insert("mirror_file", {
				"mid" : mid,
				"path": relpath,
				"status": 1,
				"fid": fid,
				"lastcheck": UpqDB().now()
				})
		except UpqDBIntegrityError:
			res = UpqDB().query("SELECT mfid FROM mirror_file WHERE mid=%s AND fid=%s" % (mid, fid))
			id = res.first()[0]
			UpqDB().query("UPDATE mirror_file SET lastcheck=NOW() WHERE mfid = %s"% (id))

	def run(self):
		dled = {}
		url = self.prefix + '/list.php'
		#print self.getlobbyversion()
		f = my_download().open(url)
		data = json.loads(str(f.read()))
		res = UpqDB().query("SELECT mid from mirror WHERE url_prefix='%s'" % self.prefix)
		mid = res.first()[0]
		for row in data:
			self.update(row, mid)
		#delete files that wheren't updated this run (means removed from mirror)
		UpqDB().query("DELETE FROM `mirror_file` WHERE `lastcheck` < NOW() - INTERVAL 1 HOUR AND mid = %s" %(mid))
Example #43
0
 def updatefile(self, fid, command):
     UpqDB().insert("sf_sync", {'fid': fid, 'command': command})
Example #44
0
    def run(self):
        if self.jobdata.has_key('fid'):  #fid set, add change to db
            fid = int(self.jobdata['fid'])
            command = 0  #default to update command
            if self.jobdata.has_key('command'):
                command = self.jobdata['command']
                if command == "update":
                    command = 0
                elif command == "delete":
                    command = 1
                else:  #unknown command
                    self.logger.error(
                        "Invalid command: %s, only update or delete are valid"
                        % (command))
                    return False
            self.updatefile(fid, command)

        username = self.jobcfg['username']
        password = self.jobcfg['password']
        proxy = ServerProxy(self.jobcfg['rpcurl'])
        if self.jobdata.has_key(
                'sid'):  #sid set, update to springfiles requested
            sid = int(self.jobdata['sid'])
        else:  #sid not set, request from springfiles
            try:
                sid = int(proxy.springfiles.getlastsyncid(username, password))
                self.logger.debug("Fetched sid from springfiles: %d" % (sid))
            except Exception as e:
                self.msg("xmlrpc springfiles.getlastsyncid() error: %s" % (e))
                return False
        if sid > 0:
            results = UpqDB().query(
                "SELECT fid, command FROM sf_sync WHERE sid>%d ORDER BY sid " %
                (sid))  #get all changes from db
        else:
            results = UpqDB().query(
                "SELECT fid FROM file WHERE status=1 ORDER BY fid"
            )  #full sync, get all fids
            self.logger.debug("springfiles returned %d, doing full sync" %
                              (sid))
        lastfid = -1
        lastcmd = -1
        requests = []
        maxsid = UpqDB().query(
            "SELECT MAX(sid) as sid FROM sf_sync").first()['sid']
        for res in results:  #fixme: use yield
            data = {}

            if sid == 0:  #full sync, all commands are update
                data['command'] = "update"
                data['metadata'] = self.getmetadata(res['fid'])
            else:
                if lastfid == res['fid'] and lastcmd == res[
                        'command']:  #skip if the same command was already done before
                    continue
                lastfid = res['fid']
                if 'command' in data:
                    lastcmd = data['command']
                if res['command'] == 1:  # delete
                    data['command'] = "delete"
                elif res['command'] == 0:  # update
                    data['metadata'] = self.getmetadata(res['fid'])
                    data['command'] = "update"
                else:
                    self.logger.error("unknown command %d for fid %d",
                                      res['command'], res['fid'])
                    continue

            self.logger.debug("adding sid %d to requests" % (res['fid']))
            data['fid'] = res['fid']
            data['sid'] = maxsid
            requests.append(data)
            if len(requests) > int(
                    self.jobcfg['maxrequests']
            ):  #more then maxrequests queued, flush them
                self.logger.debug("flushing %d requests" % (len(requests)))
                if not self.rpc_call_sync(proxy, username, password, requests):
                    return False
                requests = []
        if len(requests) > 0:  #update remaining requests
            return self.rpc_call_sync(proxy, username, password, requests)
        return True