def post(self):
if not self.is_admin():
self.redirect('/')
action = self.request.get("action")
actionLog=""
if action == "update_all":
Student.update_students()
Teacher.update_teachers()
memcache.delete('STUDENTS')
memcache.delete('TEACHERS')
actionLog="mise à jour des étudiants et enseignant"
else:
username = self.request.get("utilisateur")
user = User.find_by_username(username)
if action == "ajout":
user.is_admin = True
user.put()
actionLog="ajout de %s en tant qu'administrateur" % (user.name)
else:
user.is_admin = False
user.put()
actionLog="suppression de %s en tant qu'administrateur" % (user.name)
dateCurrent = self.get_date()
d = int(dateCurrent.strftime('%d'))
m = int(dateCurrent.strftime('%m'))
y = int(dateCurrent.strftime('%Y'))
h = int(dateCurrent.strftime('%H'))
min = int(dateCurrent.strftime('%M'))
dateCurrent = datetime.datetime(y, m, d, h, min)
ip=self.request.remote_addr
username=self.check_cookie()
Log.write_log(actionLog,ip,dateCurrent,username)
self.redirect('/admin')
def QueryUnread(self,index, boardname):
# Log.debug("QueryUnread: %s %d" % (boardname, index))
board = BoardManager.BoardManager.GetBoard(boardname)
if (board == None):
Log.error("Fail to load board %s for unread?" % boardname)
return False
entry = self.FindCacheEntry(board)
if (entry == -1):
Log.warn("cannot find cache entry for unread? %s", boardname)
return False
self._cache[entry].Update(True)
for j in range(0, BRC_MAXNUM):
cur = self._cache[entry]._list[j]
# Log.debug("read: %d" % cur)
if (cur == 0):
if (j == 0):
# Log.debug("empty bread cache")
return True
# Log.debug("reach bread cache end")
return False
if (index > cur):
# Log.debug("not found")
return True
elif (index == cur):
# Log.debug("found")
return False
return False
class Upgrade:
#---------------------------------------------------------------------------#
# Constructor
#---------------------------------------------------------------------------#
def __init__(self):
self.log = Log()
self.error = 0
self.Version = "v0.1.6"
#---------------------------------------------------------------------------#
# Get current version
#---------------------------------------------------------------------------#
def GetCurrentVersion(self):
version = "v0.0"
#Connect to MySQL
db = MySQLdb.connect(Config.DbHost, Config.DbUser, Config.DbPassword, Config.DbName)
cursor = db.cursor()
try:
#Execure SQL-Query
cursor.execute("SELECT SettingValue FROM ha_settings WHERE SettingName='Version'")
result = cursor.fetchone()
version = result[0]
except MySQLdb.Error, e:
#Log exceptions
try:
self.error = 1
self.log.error('Server', 'MySQL Error [%d]: %s' % (e.args[0], e.args[1]))
except IndexError:
self.error = 1
self.log.error('Server', 'MySQL Error: %s' % str(e))
finally:
def FetchById( id ):
"""
Fetches an individual category record from the database using the specified identifier.
Args:
id: int, the unique identifier of the category.
Returns:
A dict containing all the category fields: id, parent_id, name, code
"""
result = None
try:
Log.info(('CATEGORIES-Fetch-Id:', 'Trying to grab data from table using Id'))
query = "SELECT * FROM category WHERE id = %s;"
db.cursor.execute( query, ( id, ) )
result = db.cursor.fetchone()
Log.info(('CATEGORIES-Fetch-Id:', 'Successfully grabbed data'))
except Error as e:
Log.error(('CATEGORIES-Fetch-Id:', e))
Log.info(('CATEGORIES-Fetch-Id:Query:', query))
Log.info(('CATEGORIES-Fetch-Id:','Failed to grab data'))
return result
def post(self):
missing_students = self.request.get_all("absence")
nom_cours = self.request.get("nom_cours")
for studentName in missing_students:
student = Student.get_by_name(studentName)
# Enregistrer l'absence dans la BDD
date = self.get_date()
d = int(date.strftime('%d'))
m = int(date.strftime('%m'))
y = int(date.strftime('%Y'))
h = int(date.strftime('%H'))
min = int(date.strftime('%M'))
date = datetime.datetime(y, m, d, h, min)
Absence(studentName=studentName, group=student.get_group_string(), name_course=nom_cours,
date=date, parent=DBKey.key()).put()
dateCurrent = self.get_date()
d = int(dateCurrent.strftime('%d'))
m = int(dateCurrent.strftime('%m'))
y = int(dateCurrent.strftime('%Y'))
h = int(dateCurrent.strftime('%H'))
min = int(dateCurrent.strftime('%M'))
dateCurrent = datetime.datetime(y, m, d, h, min)
ip=ip = self.request.remote_addr
username=self.check_cookie()
action="appel du cours %s" % (nom_cours)
Log.write_log(action,ip,dateCurrent,username)
self.redirect('/')
def loadExperience(self, strOwlFile, strDesignatorFile):
logReturn = Log()
logReturn.setOwlData(self.rdrOwl.loadOwl(strOwlFile))
logReturn.setDesignatorData(self.rdrDesig.loadDesignators(strDesignatorFile))
self.addExperience(logReturn)
def getChunkBaseName1(self, poolName, objId):
Log.debug("____Get the chunks list for the object [" + str(objId) + "] and the pool[ " + str(poolName) + "]")
outdata = self.executeCmd(
"rados --pool ", [poolName, " getxattr ", '"' + objId + '"' + " ", "user.rgw.manifest"], []
)
shadow = (
outdata.replace("\x00", "")
.replace("\x07", "")
.replace("\x01", "")
.replace("\x02", "")
.replace("\x08", "")
.replace("\x03", "")
.replace("\x11", "")
.replace("\x12", "")
)
# '.index\x08\x08!.a8IqjFd0B9KyTAxmOh77aJEAB8lhGUV_\x01\x02\x01 \x08@\x07\x03_\x07cephfun\x0c'
# Log.debug("___Shadow: "+shadow)
if shadow.count("shadow") > 0:
shadow_motif = re.search("(?<=_shadow__)(\w(\-)*)+", shadow)
print "_____ shadow motif= ", shadow_motif
# chunkname=shadow_motif[shadow_motif.index('_shadow__')+9:,]
chunkname = shadow_motif.group(0)
chunkname = chunkname[0 : chunkname.index("_")]
elif shadow.count("!."):
# shadow_motif = re.search('(?<=\!\.)\w+', shadow)
# chunkname=shadow_motif.group(0)
chunkname = shadow[shadow.index("!.") + 3 : shadow.index("_ @")]
else: # The case the object has no chunk because it's not too large
chunkname = ""
Log.debug("____Chunkbasename= " + chunkname)
return chunkname
def writeToFile(keys, values, directory, filename, additional):
if not os.path.exists(directory):
os.makedirs(directory)
fo = open(directory + "/" + filename, "wb")
stringEncoding = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<resources>\n"
fo.write(stringEncoding)
for x in range(len(keys)):
if values[x] is None or values[x] == '':
Log.error("Key:" + keys[x] +
"\'s value is None. Index:" + str(x + 1))
continue
key = keys[x].strip()
value = re.sub(r'(%\d\$)(@)', r'\1s', values[x])
content = " <string name=\"" + key + "\">" + value + "</string>\n"
fo.write(content)
if additional is not None:
fo.write(additional)
fo.write("</resources>")
fo.close()
def get_bbs_sessions(self):
new_sessions = {}
lockfd = Utmp.Utmp.Lock()
try:
login = Login.Login.list_head()
seen = set()
if (login != None): # if list is not empty
while (True):
session = SessionInfo(login.get_loginid())
if (session.get_jid() in new_sessions):
new_sessions[session.get_jid()].append(session)
else:
new_sessions[session.get_jid()] = [session]
seen.add(login.get_loginid())
login = login.list_next()
if (login == Login.Login.list_head()):
break
if (login.get_loginid() in seen):
Log.warn("get_bbs_sessions(): LOOP in UtmpHead.LIST!")
break
finally:
Utmp.Utmp.Unlock(lockfd)
return new_sessions
def getPoolType(self, poolname, poolId):
Log.info("___getPoolType(poolname=" + str(poolname) + ", poolId=" + str(poolId) + ")")
outdata = self.executeCmd("ceph osd dump ", [], [poolname, " " + poolId])
pooltype = outdata.strip().split(" ")[
3
] # ['pool', '26', "'.rgw.buckets'", 'replicated', 'size', '2', 'min_size', '1', 'crush_ruleset', '0', 'object_hash', 'rjenkins', 'pg_num', '8', 'pgp_num', '8', 'last_change', '408', 'stripe_width', '0']
return pooltype
def startProject():
global projectid
db = DBConnection()
# create new project
for f in os.listdir('upload/'):
filename = f
project = Project(filename, admin.name)
projectid = project.id
# set server state
server = db.server()
server['state'] = 'activated'
server.save()
# create Logs
log = Log()
log.logProjectStart(project)
log.logFragments(project.id, project.fragments)
# request start middleware
request = requests.post("http://localhost:8000/start/"+ str(projectid))
#flash("clustering done")
return redirect(url_for('dashboard'))
def getObjectStructure() :
Log.debug("Calling getObjectStructure() method")
try :
return Response(S3ObjectCtrl(conf).getObjectStructure(),mimetype='application/json')
except S3Error , e :
Log.err(e.__str__())
return Response(e.reason, status=e.code)
class Crawler:
def __init__(self, url, chPart, offset, ds):
self.url = url
self.chPart = chPart
self.offset = offset
self.docStore = ds
self.log = Log('logfile.log')
# Baranje do server za strana, ja vrakjam sodrzinata od stranta
def request(self, pgid):
print self.url + str(pgid)
try:
page = req.get(self.url + str(pgid) + '/')
if page.content == 'Error':
self.log.log(str(time.time()) + ' - Request to file ' + str(pgid) + ' failed.')
return None
file = open(self.docStore + str(pgid) + '.html', 'w')
file.write(page.content)
file.close
except IndexError, e:
page = None
self.log.log(str(time.time()) + ' - Request to file ' + pgid + ' failed. ' + str(e.args))
return page.content
def Init(self):
if self._cache_map == None:
cachepath = User.User.CacheFile(self._userid, '')
try:
os.mkdir(cachepath, 0700)
except:
pass
entrypath = User.User.CacheFile(self._userid, 'entry')
try:
os.stat(entrypath)
except:
# Log.debug("no brc cache file for %s, creating" % self._userid)
brc = '\0' * BRC_CACHE_NUM * BrcCacheEntry.Size()
fbrc = os.open(entrypath, os.O_RDWR | os.O_CREAT, 0600)
os.write(fbrc, brc)
os.close(fbrc)
fbrc = open(entrypath, "r+b")
if (fbrc == None):
Log.error("cannot init brc cache for %s" % self._userid)
return False
self._cache_map = mmap.mmap(fbrc.fileno(), BRC_CACHE_NUM * BrcCacheEntry.Size(), prot = mmap.PROT_READ | mmap.PROT_WRITE, flags = mmap.MAP_SHARED)
fbrc.close()
if (self._cache_map == None):
Log.error("failed to mmap cache file for %s" % self.userid)
return False
self._cache = [0] * BRC_CACHE_NUM
for i in range(0, BRC_CACHE_NUM):
self._cache[i] = BrcCacheEntry(self, i)
self._cache[i].Load(self._cache_map[BrcCacheEntry.Size() * i:BrcCacheEntry.Size() * (i+1)])
return True
def getChunkSize2(self, poolName, objectid):
Log.debug("___getChunkSize(poolName=" + str(poolName) + ", objectId=" + str(objectid) + ")")
outdata = self.executeCmd('rados --pool=', [poolName, ' stat ', objectid], [])
#'.rgw.buckets/default.4651.2__shadow__0cIEZvHYuHkJ6xyyh9lwX4pj5ZsHrFD_125 mtime 1391001418, size 4194304\n'
objectsize = outdata[outdata.index('size') + 5: outdata.index('\n')]
return objectsize.rstrip()
def FetchByCode(code):
"""
Fetches an individual region record from the database using the specified region code.
Args:
code: string, the region's code.
Returns:
A dict containing all the region fields: id, parent_id, code, name
"""
result = None
try:
Log.info(("REGIONS-Fetch-Code", "Trying to grab data from table using Code"))
query = "SELECT * FROM region WHERE code = %s;"
db.cursor.execute(query, (code,))
result = db.cursor.fetchone()
Log.info((("REGIONS-Fetch-Code", "Successfully grabbed data")))
except Error as e:
Log.info(("REGIONS-Fetch-Code", e))
Log.info(("REGIONS-Fetch-Code:Query:", query))
Log.info(("REGIONS-Fetch-Code:", "Failed to grab data"))
return result
def deleteCapability(uid):
Log.debug("deleteCapability")
try:
return Response(S3Ctrl(conf).deleteCapability(uid),mimetype='application/json')
except S3Error , e:
Log.err(e.__str__())
return Response(e.reason, status=e.code)
def deleteSubuserKey(uid, subuser, key):
Log.debug("deleteSubuserKey")
try:
return Response(S3Ctrl(conf).deleteSubuserKey(uid, subuser,key),mimetype='application/json')
except S3Error , e:
Log.err(e.__str__())
return Response(e.reason, status=e.code)
def FetchByRegionCode( code ):
"""
Fetch all the shop records that belong to the region specified by the region code. This is a common use-case internally
within Oxfam.
Args:
code: string, the region code
Returns:
A list of dicts containing all the shop records that belong to the specified region (area).
"""
result = None
# While we could just do this in one SQL statement, we're going to use the Regions module so that the code is more robust.
# This can be changed if required when we look at optimization.
region = Regions.FetchByCode( code )
if region is not None:
try:
Log.info(('SHOPS-Fetch-RegionCode:', 'Trying to grab data using regionCode/Id '))
query = "SELECT * FROM shop WHERE region_id = %s;"
db.cursor.execute( query, ( region[ "id" ], ) )
result = db.cursor.fetchall()
Log.info(('SHOPS-Fetch-RegionCode:', 'Successfully grabbed data'))
except Error as e:
Log.error(('SHOPS-Fetch-RegionCode', e))
Log.info(('SHOPS-Fetch-RegionCode:Querry:', query))
Log.info(('SHOPS-Fetch-RegionCode:', 'Failed to grab data'))
return result
def addParser():
parser = OptionParser()
parser.add_option("-f", "--fileDir",
help="Xls files directory.",
metavar="fileDir")
parser.add_option("-t", "--targetDir",
help="The directory where the xml files will be saved.",
metavar="targetDir")
parser.add_option("-e", "--excelStorageForm",
type="string",
default="multiple",
help="The excel(.xls) file storage forms including single(single file), multiple(multiple files), default is multiple.",
metavar="excelStorageForm")
parser.add_option("-a", "--additional",
help="additional info.",
metavar="additional")
(options, args) = parser.parse_args()
Log.info("options: %s, args: %s" % (options, args))
return options
def parsed_headers(self):
if self.DEBUG:
Log.pdebug("Headers.parsed_headers")
if not self.headers_have_been_parsed:
self.__parse()
return self.__headers
def send(self, evt, data={}):
if self.live == False:
return
message = microjson.to_json({"evt": evt, "data": data})
Log.info("Sending message: "+message)
self._socket.send(message+"\n")
def FetchByCode( code ):
"""
Fetches an individual shop record from the database using the specified shop code.
Args:
code: string, the shop's code.
Returns:
A dict containing all the shop fields: id, region_id, code, name
"""
result = None
try:
Log.info(('SHOPS-Fetch-Code:', 'Trying to grab data from table using Code'))
query = "SELECT * FROM shop WHERE code = %s;"
db.cursor.execute( query, ( code, ) )
result = db.cursor.fetchone()
Log.info(('SHOPS-Fetch-Code:', 'Successfully grabbed data'))
except Error as e:
Log.error(('SHOPS-Fetch-Code:', e))
Log.info(('SHOPS-Fetch-Code:Query:',query))
Log.info(('SHOPS-Fetch-Code:', 'Failed to grab data'))
return result
def _start_socket(self, port):
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.connect(('', port))
self._socket.setblocking(0)
self.live = True;
Log.info("Server started")
def loadLog(self, strPath):
log = Log()
log.setOwlData(self.rdrOwl.loadOwl(strPath + "/cram_log.owl"))
log.setDesignatorData(self.rdrDesig.loadDesignators(strPath + "/logged_designators.json"))
return log
def write(fName, pCollection):
Log.trace(__class__, "write()")
try:
file = open(fName, 'w')
SEP = ","
EMB = "\""
# write headerNames first
sep = ""
for col in pCollection.getHeaderNames():
file.write(sep + EMB + col + EMB)
if(sep != SEP):
sep = SEP
file.write("\n")
# write persons
for person in pCollection.getPersons():
sep = ""
for col in pCollection.getHeaderNames():
file.write(sep + EMB + person.getAttribute(col) + EMB)
if(sep != SEP):
sep = SEP
file.write("\n")
file.close()
return True
except IOError:
Log.error(__class__, "IOError with file > " + fName)
def __init__(self):
self.live = False
try:
self._start_socket(1111)
except Exception, err:
Log.info(err)
Log.info("Accept called")
class LRGParser(HTMLParser.HTMLParser):
def __init__(self, downloadFileControl, toAddPassword, linkType):
HTMLParser.HTMLParser.__init__(self)
self.valList = []
self.done = False
self.toAddPassword = toAddPassword
self.linkType = linkType
self.downloadFileControl = downloadFileControl
self.linksDict = {}
self.log = Log()
def feed(self, htmlBody):
try:
HTMLParser.HTMLParser.feed(self, htmlBody)
except HTMLParser.HTMLParseError, e:
self.log.debug('LRGParser HTMLParser.HTMLParseError', e)
if self.linkType == URLCASH:
tmpHTMLBody = htmlBody
pos = tmpHTMLBody.find(URLCASH_IFRAME)
if pos != -1:
tmpBody = tmpHTMLBody[pos + len(URLCASH_IFRAME):]
rapidUrl = tmpBody[:tmpBody.find(SINGLE_QUOTE)]
self.log.debug(rapidUrl)
self.linksDict[rapidUrl] = rapidUrl
self.downloadFileControl.processTag(self.valList, self.linksDict)
else:
self.downloadFileControl.reportError('Cannot find rapidshare links')
def write(fName, pCollection): if(fName.endswith(CSV.SUFFIX)): CSV.write(fName, pCollection) elif(fName.endswith(VCARD.SUFFIX)): VCARD.write(fName, pCollection) else: Log.error(__class__, "write() - File format not supported!")
def read(fName): Log.trace(__class__, "read()") pCollection = PersonCollection() try: file = open(fName, 'r') isHeader = True #SEP = ',*' SEP = "[\,,\s]*" EMB = "\"" regEx = re.compile(EMB + '([^' + EMB + ']*)' + EMB + SEP) for line in file: i = 0 person = Person() for col in regEx.findall(line): if(isHeader): pCollection.addHeader(col) #self._headerNames.append(col) else: person.setAttribute(pCollection.getHeaderNames()[i], col) i += 1 if(isHeader): isHeader = False else: pCollection.addPerson(person) file.close() return pCollection except IOError: Log.error(__class__, "IOError with file > " + fName) return None
def deleteSubuser(self, uid, subuser):
Log.debug("delete subuser " + subuser + " for user with uid " + uid)
return S3User.deleteSubuser(uid, subuser, self.getAdminConnection())
def createSubuser(self, uid):
Log.debug("create subuser for user with uid " + uid)
jsonform = request.form['json']
return S3User.createSubuser(uid, jsonform, self.getAdminConnection())
def removeUserKey(self, uid, key):
Log.debug("remove key for user with uid " + uid)
return S3User.removeKey(key, self.getAdminConnection())
def setUp(self):
self.log = Log()
def __init__(self, clientInfo):
self.clientInfo = clientInfo
log = Log()
self.logging = log.GetLogging()
conn = MongoClient("localhost", 27017)
self.db = conn["webSite"]
def getArticles(self, type, limit, mark):
cursor = self.getCursor(mark)
cursor_doc = cursor.find({
"type_id": type
}).sort([("rankingScore", pymongo.ASCENDING),
("date", pymongo.ASCENDING)]).limit(limit)
array = []
for doc in cursor_doc:
del doc["_id"]
array.append(doc)
return array
def getCursor(self, mark):
if mark == "A":
cursor = self.db.articlesA
else:
cursor = self.db.articlesB
return cursor
if __name__ == '__main__':
now = datetime.datetime.now()
log = Log(now)
service = MongoService()
dict = service.setArticles()
print len(dict)
print type(dict)
def listUsers(self):
Log.debug("list users from rgw api")
return S3User.list(self.getAdminConnection())
def getUserBuckets(self, uid):
Log.debug("getBuckets for uid " + uid)
jsonform = None
return S3User.getBuckets(uid, jsonform, self.getAdminConnection())
num_layers=num_layers,
learning_rate=learning_rate,
train_mode=True)
log.info("create seq2seq model")
log.info(str(seq2seq.params_set()))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
checkpoint = tf.train.latest_checkpoint(model_save_path)
if checkpoint:
seq2seq.load(sess, checkpoint)
log.info("load model form:" + checkpoint)
for index in range(epochs):
for batch_i, (targets_batch, sources_batch, targets_lengths,
sources_lengths) in enumerate(
char_data.get_batches(batch_size)):
loss = seq2seq.train(sess, sources_batch, sources_lengths,
targets_batch, targets_lengths)
log.info("epochs:{}/{} loss:{} ".format(index, epochs, loss))
seq2seq.save(sess, model_save_path + model_name)
log.info("save model:" + model_save_path + model_name)
if __name__ == '__main__':
check_dir()
log = Log(train_log_name)
train(log)
def getObjectStructure(self):
startdate = datetime.now()
print(
str(startdate) + ' -Calling method getObjectStructure() begins <<')
print " __request", request
objectId = request.args.get('objectId')
bucketname = request.args.get('bucketName')
osd_dump = self.getOsdDump()
# objectIdd=request.form['objectId']
#bucketnamee=request.form['bucketName']
Log.debug("__getS3Object(objectId=" + str(objectId) +
", bucketName= " + str(bucketname) + ")")
#Log.debug("getS3Object(objectIdd="+str(objectIdd)+", bucketNamee= "+str(bucketnamee)+")")
#Retrieve the bucketId using the bucket name
bucketId = self.getBucketInfo(bucketname)["bucketid"]
#Get the pool name using the
poolname = self.getBucketInfo(bucketname)["poolname"]
#Retrieve the pool id
poolid = self.getPoolId(poolname)
#poolname=getPoolName(bucketName)
extended_objectId = bucketId + "_" + objectId
#Retrieve the user.rgw.manifest that contains the chunks list for the object
#usermnf=self.getUserRgwManifest(poolname,extended_objectId)
#Retrieve the chunk base name in the user.rgw.manifest attribute
chunkbasename = self.getChunkBaseName(poolname, extended_objectId)
print '__Chunk base name: ', chunkbasename
if len(chunkbasename): #chek if there is chunk por not for the object
#Retrieve the chunks list of the object
chunks = self.getChunks(bucketId, poolname, objectId,
chunkbasename)
chunks.append(extended_objectId
) #Add the last object that is around 512.0 kb
else:
chunks = [extended_objectId]
print "__Chunks list", chunks
#bucketInfo=self.getBucketInfo(bucketId)
chunklist = []
pgs = []
osds = []
osdids = []
pgid4osd = []
for chunk in chunks:
if len(chunk) > 0:
print 'Chunk= ', chunk
chunksize = self.getChunkSize(poolname, chunk)
pgid = self.getPgId(poolname, ' ' + chunk)
c = Chunk(chunk, chunksize, pgid[0])
chunklist.append(c)
if pgid4osd.count(pgid[1]) == 0:
pgid4osd.append(pgid[1])
if pgid4osd.count(pgid[0]) == 0:
pgid4osd.append(pgid[0])
#Create the PG for this chunk
#ef __init__(self,pgid,state,acting, up, acting_primary, up_primary):
pginfos = self.getOsdMapInfos(pgid[1])
pg = PG(pgid[0], pginfos['state'], pginfos['acting'],
pginfos['up'], pginfos['acting_primary'],
pginfos['up_primary'])
# print(pg.dump())
pgs.append(pg) #Append the PG in the pgs list
# print "____ OSD List for PG ", pgid[1],self.getOsdsListForPg(pgid[1])
for pgid in pgid4osd:
for id in self.getOsdsListForPg(
pgid): #sortir la boucle pour les pg
if osdids.count(id) == 0:
osdids.append(
id) #construct the list of the OSD to be displayed
#Log.debug("Total number of chunks retrived:"+str(nbchunks))
# print "_____osds list=",osdids
for osdid in osdids: #Loop the OSD list and retrieve the osd and add it in the osds list fot the S3 object
osd = self.getOsdInfos(osd_dump, osdid)
#print(osd.dump())
osds.append(osd)
s3object = S3Object(extended_objectId, bucketname, bucketId, poolid,
poolname, self.getPoolType(poolname, poolid),
self.getChunkSize(poolname, extended_objectId),
chunklist, pgs, osds)
print(s3object.dump())
duration = datetime.now() - startdate
Log.info(
str(datetime.now()) +
' ___Calling method getObjectStructure() end >> duration= ' +
str(duration.seconds))
return s3object.dump()
def getPoolId(self, poolname):
Log.info("___getPoolId(poolname=" + str(poolname) + ")")
outdata = self.executeCmd('ceph osd pool stats ', [poolname], [])
poolid = outdata.strip().split('\n')[0].split(' id ')[
1] #['pool .rgw.buckets', ' 16']
return poolid
def deleteSubuserKey(self, uid, subuser, key):
Log.debug("delete key " + key + " for subuser " + subuser +
" for user with uid " + uid)
return S3User.deleteSubuserKey(uid, subuser, key,
self.getAdminConnection())
def __iter__(self):
'''
Returns a generator of Log objects within self.srcpath.
'''
return (Log(logfile)
for logfile in self.srcdir.glob('*' + self.srcsuff))
def is_location_in_position(self, location):
if location not in self.position:
Log.info(f'position:{location} return')
return False
def createUser(self):
Log.debug("user creation")
jsonform = request.form['json']
return S3User.create(jsonform, self.getAdminConnection())
def __init__(self, repoEngine, isOpenLog=True):
self.repoEngine = repoEngine
self.log = Log(isOpenLog)
return
import sys
print(r"""
$$\ $$\
$$ | \__|
$$$$$$\ $$$$$$\ $$$$$$\ $$$$$$\ $$$$$$\$$$$\ $$\ $$$$$$$\
\____$$\ $$ __$$\\_$$ _| $$ __$$\ $$ _$$ _$$\ $$ |$$ _____|
$$$$$$$ |$$ | \__| $$ | $$$$$$$$ |$$ / $$ / $$ |$$ |\$$$$$$\
$$ __$$ |$$ | $$ |$$\ $$ ____|$$ | $$ | $$ |$$ | \____$$\
\$$$$$$$ |$$ | \$$$$ |\$$$$$$$\ $$ | $$ | $$ |$$ |$$$$$$$ |
\_______|\__| \____/ \_______|\__| \__| \__|\__|\_______/
Personal identification log book (c) 2021
""")
print("[1] Create Case")
print("[2] Find Cases")
print("[3] Update Case")
print("[4] About")
# Convert to integer
inputText = int(input("Type number: "), 10)
obj = Log()
if (inputText == 1):
obj.create_log(date.today())
elif (inputText == 2):
obj.find_log()
elif (inputText == 3):
obj.update()
else:
print("about")
class AppLauncherApplet(Budgie.Applet):
# Budgie.Applet is in fact a Gtk.Bin
def __init__(self, uuid):
self.TAG = "budgie-app-launcher.AppLauncher"
self.APPINDICATOR_ID = "io_serdarsen_github_budgie_app_launcher"
self.APPS_ID = "gnome-applications.menu"
self.log = Log("budgie-app-launcher")
self.sortHelper = SortHelper()
self.filterHelper = FilterHelper()
self.localHelper = LocaleHelper()
self.iconSize = 24
self.showOnPanel = 0
self.tree = None # GMenu.Tree
self.appButtonsContainer = None # Gtk.ListBox
self.contentScroll = None # Gtk.ScrolledWindow
self.allAppsContentScroll = None
self.currentMenuButton = None
self.menuButtons = []
self.allApps = []
self.filteredActiveApps = []
self.activeApps = []
self.inactiveApps = []
self.appLimitOnPanel = 10
self.manager = None
self.popover = None
self.popoverHeight = 0
self.popoverWidth = 300
self.popoverHeight = 510
Budgie.Applet.__init__(self)
self.iconSize = self.localHelper.retriveIconSize()
self.showOnPanel = self.localHelper.retriveShowOnPanel()
self.buildIndicator()
self.buildPopover()
self.buildStack()
self.loadAllApps()
self.loadAppButtons()
self.loadPanelButtons()
self.vertical = False
def do_panel_position_changed(self, position):
# wait for signal, change orientation if it occurs
check_or = any(
[position == Budgie.PanelPosition(pos) for pos in [8, 16]])
self.vertical = True if check_or else False
self.reload_elements()
def reload_elements(self):
# on orientation change of the panel, set applet accordingly
self.panelButtonsContainer.destroy()
self.panelButtonsContainer = Gtk.VBox() if self.vertical \
else Gtk.HBox()
self.indicatorBox.add(self.panelButtonsContainer)
self.update()
####################################
# build START
####################################
def buildIndicator(self):
self.indicatorBox = Gtk.EventBox()
self.panelButtonsContainer = Gtk.VBox()
self.indicatorBox.add(self.panelButtonsContainer)
self.add(self.indicatorBox)
def buildPopover(self):
self.popover = Budgie.Popover.new(self.indicatorBox)
self.popover.set_default_size(self.popoverWidth, self.popoverHeight)
self.popover.get_child().show_all()
self.show_all()
def buildStack(self):
self.stack = Gtk.Stack()
self.stack.set_homogeneous(False)
self.stack.set_transition_type(
Gtk.StackTransitionType.SLIDE_LEFT_RIGHT)
self.popover.add(self.stack)
self.buildStackPage1()
self.buildStackPage2()
def buildStackPage1(self):
# page 1
page1 = Gtk.Box(Gtk.Orientation.VERTICAL, 0)
page1.border_width = 0
self.stack.add_named(page1, "page1")
page1.get_style_context().add_class("budgie-menu")
# page 1 content
page1InnerBox = Gtk.VBox()
page1.pack_start(page1InnerBox, True, True, 0)
titleBox = Gtk.HBox()
page1InnerBox.pack_start(titleBox, False, False, 0)
self.setMargins(titleBox, 3, 3, 3, 0)
self.searchEntry = Gtk.SearchEntry()
titleBox.pack_start(self.searchEntry, True, True, 0)
self.searchEntry.connect("search-changed", self.searchEntryOnChange)
self.searchEntry.connect("activate", self.searchEntryOnActivate)
editButton = EditButton("Edit")
editButton.connect("clicked", self.editButtonOnClick)
titleBox.pack_end(editButton, False, False, 0)
self.contentScroll = Gtk.ScrolledWindow(None, None)
page1InnerBox.pack_start(self.contentScroll, True, True, 0)
self.contentScroll.set_overlay_scrolling(True)
self.contentScroll.set_policy(Gtk.PolicyType.NEVER,
Gtk.PolicyType.AUTOMATIC)
self.appButtonsContainer = Gtk.ListBox()
self.contentScroll.add(self.appButtonsContainer)
# self.content.row_activated.connect(on_row_activate)
self.appButtonsContainer.set_selection_mode(Gtk.SelectionMode.NONE)
# placeholder in case of no results
placeholder = Gtk.Label("App Launcher")
placeholder.use_markup = True
placeholder.get_style_context().add_class("dim-label")
placeholder.show()
placeholder.margin = 6
self.appButtonsContainer.valign = Gtk.Align.START
self.appButtonsContainer.set_placeholder(placeholder)
def buildStackPage2(self):
# page 2
page2 = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=0)
page2.border_width = 0
self.setMargins(page2, 10, 3, 3, 3)
self.stack.add_named(page2, "page2")
# page 2 toolbar
page2Toolbar = Gtk.HBox()
page2.pack_start(page2Toolbar, False, False, 0)
# page 2 toolbar content
# cancel
backButton = DirectionalButton("Back", Gtk.PositionType.LEFT)
page2Toolbar.pack_start(backButton, True, True, 0)
backButton.connect("clicked", self.backButtonOnClick)
# up
self.upButton = ArrowButton(Gtk.PositionType.TOP)
page2Toolbar.pack_start(self.upButton, True, True, 0)
self.upButton.connect("clicked", self.upButtonOnClick)
self.upButton.set_sensitive(False)
# down
self.downButton = ArrowButton(Gtk.PositionType.BOTTOM)
page2Toolbar.pack_start(self.downButton, True, True, 0)
self.setMargins(page2Toolbar, 0, 3, 0, 0)
self.downButton.connect("clicked", self.downButtonOnClick)
self.downButton.set_sensitive(False)
# done
self.doneButton = Gtk.Button("Apply")
self.doneButton.get_style_context().add_class("flat")
page2Toolbar.pack_end(self.doneButton, True, True, 0)
self.doneButton.connect("clicked", self.doneButtonOnClick)
# page 2 toolbar 2
page2Toolbar2 = Gtk.HBox()
page2.pack_start(page2Toolbar2, False, False, 0)
self.deselectButton = SelectButton(False)
page2Toolbar2.pack_start(self.deselectButton, False, False, 0)
self.deselectButton.addOnClickMethod(self.deselectButtonOnClick)
self.selectButton = SelectButton(True)
page2Toolbar2.pack_end(self.selectButton, False, False, 0)
self.selectButton.addOnClickMethod(self.selectButtonOnClick)
self.setMargins(page2Toolbar2, 0, 3, 0, 0)
# page 2 allAppsContainer
self.allAppsContainer = Gtk.VBox()
page2.pack_start(self.allAppsContainer, True, True, 0)
self.allAppsContentScroll = Gtk.ScrolledWindow(None, None)
self.allAppsContainer.pack_start(self.allAppsContentScroll, True, True,
0)
self.allAppsContentScroll.set_overlay_scrolling(True)
self.allAppsContentScroll.set_policy(Gtk.PolicyType.NEVER,
Gtk.PolicyType.AUTOMATIC)
self.menuButtonsContainer = Gtk.VBox()
self.setMargins(self.menuButtonsContainer, 0, 0, 0, 7)
self.allAppsContentScroll.add(self.menuButtonsContainer)
# page 2 bottom bar
page2BottomBar = Gtk.VBox()
page2.pack_start(page2BottomBar, False, False, 0)
self.setMargins(page2BottomBar, 10, 3, 3, 3)
# page2BottomBar show panel Container
page2BottomBarShowPanelContainer = Gtk.HBox()
page2BottomBar.pack_start(page2BottomBarShowPanelContainer, False,
False, 0)
showOnPanelLabel = Gtk.Label("Show on panel", xalign=0)
page2BottomBarShowPanelContainer.pack_start(showOnPanelLabel, True,
True, 0)
showOnPanelLabel.get_style_context().add_class("dim-label")
self.showOnPanelSpinButton = Gtk.SpinButton()
self.showOnPanelSpinButton.set_adjustment(
Gtk.Adjustment(0, 0, 900, 1, 10, 0))
self.showOnPanelSpinButton.set_value(self.showOnPanel)
page2BottomBarShowPanelContainer.pack_start(self.showOnPanelSpinButton,
False, False, 0)
func = self.showOnPanelSpinButtonOnValueChange
self.showOnPanelSpinButton.connect("value-changed", func)
# page2BottomBar icon size Container
page2BottomBarIconSizeContainer = Gtk.HBox()
page2BottomBar.pack_start(page2BottomBarIconSizeContainer, False,
False, 0)
iconSizeLabel = Gtk.Label("Icon size", xalign=0)
page2BottomBarIconSizeContainer.pack_start(iconSizeLabel, True, True,
0)
iconSizeLabel.get_style_context().add_class("dim-label")
iconSizeSpinButton = Gtk.SpinButton()
iconSizeSpinButton.set_adjustment(Gtk.Adjustment(0, 16, 512, 1, 10, 0))
iconSizeSpinButton.set_value(self.iconSize)
page2BottomBarIconSizeContainer.pack_start(
iconSizeSpinButton,
False,
False,
0,
)
iconSizeSpinButton.connect(
"value-changed",
self.iconSizeSpinButtonOnValueChange,
)
def openStackPage1(self):
self.stack.set_visible_child_name("page1")
def openStackPage2(self):
self.stack.set_visible_child_name("page2")
def indicatorBoxOnPress(self, box, e):
self.openStackPage1()
if e.button != 1:
return Gdk.EVENT_PROPAGATE
if self.popover.get_visible():
self.popover.hide()
else:
self.update()
self.manager.show_popover(self.indicatorBox)
return Gdk.EVENT_STOP
def editButtonOnClick(self, editButton):
self.currentMenuButton = None
self.updateSensitiveUpDownButtons()
self.doneButton.set_sensitive(False)
self.update()
self.openStackPage2()
def upButtonOnClick(self, button):
if self.currentMenuButton in self.menuButtons:
newIndex = self.menuButtons.index(self.currentMenuButton) - 1
if newIndex >= 0:
self.menuButtonsContainer.reorder_child(
self.currentMenuButton, newIndex)
self.menuButtons.remove(self.currentMenuButton)
self.menuButtons.insert(newIndex, self.currentMenuButton)
self.activeApps.remove(self.currentMenuButton.getApp())
self.activeApps.insert(newIndex,
self.currentMenuButton.getApp())
self.allApps.remove(self.currentMenuButton.getApp())
self.allApps.insert(newIndex, self.currentMenuButton.getApp())
self.updateAllAppsIndexes()
self.updateSensitiveUpDownButtons()
def downButtonOnClick(self, button):
if self.currentMenuButton in self.menuButtons:
newIndex = self.menuButtons.index(self.currentMenuButton) + 1
self.menuButtonsContainer.reorder_child(self.currentMenuButton,
newIndex)
self.menuButtons.remove(self.currentMenuButton)
self.menuButtons.insert(newIndex, self.currentMenuButton)
self.activeApps.remove(self.currentMenuButton.getApp())
self.activeApps.insert(newIndex, self.currentMenuButton.getApp())
self.allApps.remove(self.currentMenuButton.getApp())
self.allApps.insert(newIndex, self.currentMenuButton.getApp())
self.updateAllAppsIndexes()
self.updateSensitiveUpDownButtons()
def backButtonOnClick(self, button):
self.update()
self.openStackPage1()
def doneButtonOnClick(self, button):
self.localHelper.saveApps(self.activeApps)
self.loadAppButtons()
self.loadPanelButtons()
self.loadMenuButtons()
self.allAppsContentScroll.get_vadjustment().set_value(0)
self.doneButton.set_sensitive(False)
self.showOnPanelSpinButton.get_adjustment().set_upper(
len(self.activeApps))
self.showOnPanelSpinButton.get_adjustment().set_value(self.showOnPanel)
def selectButtonOnClick(self, selectButton):
self.selectAll()
def deselectButtonOnClick(self, deselectButton):
self.deselectAll()
def menuButtonOnToggle(self, toggleButton, *data):
self.doneButton.set_sensitive(True)
menuButton = data[0]
# if current menu button toggled again set current None
if self.currentMenuButton is not None and \
self.currentMenuButton is menuButton:
self.currentMenuButton = None
self.updateSensitiveUpDownButtons()
return
# toggle false old current button
if self.currentMenuButton is not None and \
self.currentMenuButton is not menuButton:
self.currentMenuButton.setToggled(False)
# set new currentMenuButton
self.currentMenuButton = menuButton
self.updateSensitiveUpDownButtons()
def menuButtonOnCheck(self, checkButton, *data):
self.doneButton.set_sensitive(True)
menuButton = data[0]
app = menuButton.getApp()
if not app.getActive():
app.setActive(True)
if app in self.inactiveApps:
self.inactiveApps.remove(app)
self.activeApps.append(app)
else:
app.setActive(False)
if app in self.activeApps:
self.activeApps.remove(app)
self.inactiveApps.insert(0, app)
self.inactiveApps = self.sortHelper.sortedAppsByName(self.inactiveApps)
self.allApps = self.activeApps + self.inactiveApps
self.updateAllAppsIndexes()
self.updateSensitiveSelectButtons()
def showOnPanelSpinButtonOnValueChange(self, spinButton):
# print("value changed : %s" % int(spinButton.get_value()))
self.showOnPanel = int(spinButton.get_value())
self.localHelper.saveShowOnPanel(self.showOnPanel)
self.update()
self.panelButtonsContainer.set_size_request(0, 0)
spinButton.get_adjustment().set_upper(len(self.activeApps))
def iconSizeSpinButtonOnValueChange(self, spinButton):
# print("value changed : %s" % int(spinButton.get_value()))
self.iconSize = int(spinButton.get_value())
self.localHelper.saveIconSize(self.iconSize)
self.update()
def searchEntryOnChange(self, searchEntry):
text = searchEntry.get_text().strip()
self.filteredActiveApps = self.filterHelper.filteredAppsByName(
self.activeApps, self.showOnPanel, text)
if text is "":
self.loadAppButtons()
else:
self.loadFilteredActiveAppButtons()
def treeOnChange(self, tree):
# self.log.d(self.TAG, "treeOnChange")
self.tree = None
self.update()
self.localHelper.saveApps(self.activeApps)
def searchEntryOnActivate(self, searchEntry):
if len(self.filteredActiveApps) != 0:
info = self.filteredActiveApps[0].getInfo()
if info is not None:
self.hidePopover()
info.launch(None, None)
def update(self):
# self.popover.resize(self.popoverWidth, self.popoverHeight)
self.searchEntry.set_text("")
self.loadAllApps()
self.loadAppButtons()
self.loadMenuButtons()
self.loadPanelButtons()
self.updateSensitiveSelectButtons()
self.popover.set_size_request(self.popoverWidth, self.popoverHeight)
self.popover.get_child().show_all()
self.show_all()
def do_update_popovers(self, manager):
self.manager = manager
self.manager.register_popover(self.indicatorBox, self.popover)
def updateSensitiveUpDownButtons(self):
if self.currentMenuButton is not None:
currentMenuButtonIndex = self.menuButtons.index(
self.currentMenuButton)
activeAppsLenght = len(self.activeApps)
if currentMenuButtonIndex is 0:
self.upButton.set_sensitive(False)
self.downButton.set_sensitive(True)
elif currentMenuButtonIndex is activeAppsLenght - 1:
self.upButton.set_sensitive(True)
self.downButton.set_sensitive(False)
else:
self.upButton.set_sensitive(True)
self.downButton.set_sensitive(True)
else:
self.upButton.set_sensitive(False)
self.downButton.set_sensitive(False)
def updateSensitiveSelectButtons(self):
if len(self.activeApps) is 0:
self.selectButton.set_sensitive(True)
self.deselectButton.set_sensitive(False)
elif len(self.activeApps) > 0:
self.selectButton.set_sensitive(True)
self.deselectButton.set_sensitive(True)
else:
self.selectButton.set_sensitive(False)
self.deselectButton.set_sensitive(True)
def updateAllAppsIndexes(self):
tempApps = []
for app in self.allApps:
app.setIndex(self.allApps.index(app))
tempApps.append(app)
self.allApps = tempApps
def hidePopover(self):
if (self.popover is not None):
if self.popover.get_visible():
self.popover.hide()
def loadAllApps(self):
# self.log.d(self.TAG, "loadAllApps")
self.iconSize = self.localHelper.retriveIconSize()
self.showOnPanel = self.localHelper.retriveShowOnPanel()
# reset all apps lists
self.allApps = []
self.activeApps = []
self.inactiveApps = []
# load our active apps data from local
self.activeAppsDict = self.localHelper.retriveAppsDict()
# load all apps from system all applications
appsDict = {}
self.loadAndExtractApps(appsDict)
# sort and combine active and inactiveApps to build allApps
self.activeApps = self.sortHelper.sortedAppsByIndex(self.activeApps)
self.inactiveApps = self.sortHelper.sortedAppsByName(self.inactiveApps)
self.allApps = self.activeApps + self.inactiveApps
def loadAndExtractApps(self, appsDict, treeRoot=None):
# self.log.d(self.TAG, "loadAndExtractApps")
if self.tree is None:
# self.log.d(self.TAG, "loadAndExtractApps self.tree is None")
self.tree = GMenu.Tree.new(self.APPS_ID,
GMenu.TreeFlags.SORT_DISPLAY_NAME)
self.tree.connect("changed", self.treeOnChange)
try:
self.tree.load_sync()
except Exception as e:
self.log.e(self.TAG, Error.ERROR_8011, e)
if treeRoot is None:
root = self.tree.get_root_directory()
else:
root = treeRoot
it = None
if root is not None:
it = root.iter()
if it is not None:
while True:
treeItemType = it.next()
if treeItemType is GMenu.TreeItemType.INVALID:
break
if treeItemType is GMenu.TreeItemType.DIRECTORY:
dir = it.get_directory()
# self.log.d(self.TAG, "loadAppList dir %s : " % dir)
self.loadAndExtractApps(appsDict, dir)
elif treeItemType is GMenu.TreeItemType.ENTRY:
info = it.get_entry().get_app_info()
# self.log.d(self.TAG, info.get_display_name())
id = info.get_id()
if id not in appsDict:
appsDict[id] = info
if id in self.activeAppsDict:
app = self.activeAppsDict[id]
app.setId(id)
app.setInfo(info)
self.activeApps.insert(app.getIndex(), app)
else:
app = App(id, info.get_display_name(), False)
app.setInfo(info)
self.inactiveApps.append(app)
def loadFilteredActiveAppButtons(self):
if self.appButtonsContainer is not None: # empty allAppsContent
for appButton in self.appButtonsContainer.get_children():
appButton.destroy()
for app in self.filteredActiveApps:
row = Gtk.HBox()
self.appButtonsContainer.add(row)
appButton = AppButton(app, 24, self.popover)
row.pack_start(appButton, True, True, 0)
appButton.show_all()
row.show_all()
self.appButtonsContainer.show_all()
def loadAppButtons(self):
if self.appButtonsContainer is not None: # empty allAppsContent
for appButton in self.appButtonsContainer.get_children():
appButton.destroy()
counter = 0
for app in self.allApps:
if counter >= self.showOnPanel and app.getActive():
row = Gtk.HBox()
self.appButtonsContainer.add(row)
appButton = AppButton(app, 24, self.popover)
row.pack_start(appButton, True, True, 0)
appButton.show_all()
row.show_all()
counter += 1
self.appButtonsContainer.show_all()
def loadPanelButtons(self):
if self.panelButtonsContainer is not None: # empty allAppsContent
for panelButton in self.panelButtonsContainer.get_children():
panelButton.destroy()
# add the applet's button as first item
appletbutton = Gtk.Button()
applet_icon = Gtk.Image.new_from_icon_name(
"budgie-app-launcher-applet-symbolic", Gtk.IconSize.MENU)
appletbutton.set_image(applet_icon)
appletbutton.set_relief(Gtk.ReliefStyle.NONE)
self.panelButtonsContainer.add(appletbutton)
appletbutton.connect("button-press-event", self.indicatorBoxOnPress)
counter = 0
for app in self.allApps:
if counter < self.showOnPanel and app.getActive():
panelButton = PanelButton(app, self.iconSize, self.popover)
self.panelButtonsContainer.add(panelButton)
counter += 1
self.panelButtonsContainer.show_all()
def loadMenuButtons(self):
self.menuButtons = []
if self.menuButtonsContainer is not None:
for menuButton in self.menuButtonsContainer.get_children():
menuButton.destroy()
for app in self.allApps:
menuButton = MenuButton(app, 24)
menuButton.setChecked(app.getActive())
menuButton.setToggButtonSensitive(app.getActive())
menuButton.addOnToggleMethod(self.menuButtonOnToggle)
menuButton.addOnCheckMethod(self.menuButtonOnCheck)
menuButton.show_all()
self.menuButtons.append(menuButton)
self.menuButtonsContainer.add(menuButton)
self.menuButtonsContainer.show_all()
def setMargins(self, widget, top, bottom, left, right):
widget.set_margin_top(top)
widget.set_margin_bottom(bottom)
widget.set_margin_left(left)
widget.set_margin_right(right)
def selectAll(self):
self.doneButton.set_sensitive(True)
for menuButton in self.menuButtons:
if not menuButton.getChecked():
menuButton.setChecked(True)
def deselectAll(self):
self.doneButton.set_sensitive(True)
for menuButton in self.menuButtons:
if menuButton.getChecked():
menuButton.setChecked(False)
# -*- coding: utf-8; Mode: python; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
# ex: set softtabstop=4 tabstop=4 shiftwidth=4 expandtab fileencoding=utf-8:
import re
import string
import copy
from Currency_v2 import Currency
from Database import Database, DatabaseObject
from Configuration import Configuration
from Constants import Constants
from Log import Log
import types
from DateTime import DateTime
_logger = Log.getLogger(__name__)
class NewsDocument(DatabaseObject.Document):
def __init__(self, document):
DatabaseObject.Document.__init__(self, document)
class News(DatabaseObject):
def __init__(self, database):
DatabaseObject.__init__(self, NewsDocument, database,
Configuration.newsDesignName())
@staticmethod
def database():
return Database.news(
server_connection_pool=True,
server_cache_count_min=10,
def consumption(self):
Log.customer_log('CUSTOMER: Start This Thread......')
while True:
Log.customer_log(
'-----------------------------------------------------------------------------------------'
)
try:
Log.customer_log('CUSTOMER: Start......')
submit = Manufacturer.getInstance().get_queue_head()
if submit is None:
Log.customer_log('CUSTOMER: None Data')
time.sleep(3)
continue
Log.customer_log('CUSTOMER: The submit code name is : ' +
str(submit.get_code_name()))
# TODO 评测中的编译
# 若成功则继续执行，否则回写代码
if self.compile(submit):
Log.customer_log('CUSTOMER: Compile......')
question = OJDBA.get_question(submit.get_question_id())
test_datas = OJDBA.get_test_data_by_question_id(
submit.get_question_id())
# 若测试案例小于系统定的测试次数，则只运行测试案例的数量，否则进行系统测试次数
test_time = len(test_datas) if len(
test_data
) < DATA.JUDGEMENT_TIMES else DATA.JUDGEMENT_TIMES
test_datas_length = len(test_datas)
accept_total = 0 # AC的总次数
run_time_total = 0 # 程序运行次数
# 评测程序5次并且每次的结果都是AC，则将其写回数据库
# TODO 评测过程
for i in range(0, test_time):
Log.customer_log(
'CUSTOMER: ------------------------------------------------'
)
# get a random position and choose a test Data by this position
test_data_position = random.randint(
0, test_datas_length - 1)
test_data = test_datas[test_data_position]
test_datas[test_data_position] = test_datas[
test_datas_length - 1]
test_datas_length -= 1
# run the program and get the result
Log.customer_log('CUSTOMER: Start......')
result = self.run_program(
submit.get_code_name(), submit.get_language_type(),
question, test_data.get_test_data_by_list())
Log.customer_log(
'CUSTOMER: the test data list' +
str(test_data.get_result_data_by_list()))
Log.customer_log('CUSTOMER: \n' + result)
# access data base and update the submit result
if result is not None:
Log.customer_log('CUSTOMER: Judgement......')
run_time = AnalysisResult.analysis_time(result)
run_time_total += float(run_time.replace('ms', ''))
analysis_result = AnalysisResult.analysis(
result, test_data.get_result_data_by_list())
Log.customer_log('CUSTOMER: AnalysisResult is ' +
analysis_result)
if analysis_result != DATA.ACCEPT:
OJDBA.update_by_result(analysis_result,
submit.get_code_name())
break
else:
accept_total += 1
else:
Log.customer_log('CUSTOMER: Run timer Error')
OJDBA.update_runtime_error(submit.get_code_name())
# if acceptTotal is 5, update data base the result is ac
if accept_total == 5:
OJDBA.update_accepted(
code_name=submit.get_code_name(),
run_time=(
str(run_time_total / DATA.JUDGEMENT_TIMES) +
'MS'))
else:
Log.customer_log('CUSTOMER: Compile Error')
OJDBA.update_compiler_error(submit.get_code_name())
Log.customer_log('CUSTOMER: Over......')
except Exception, e:
OJDBA.update_runtime_error(submit.get_code_name())
print 'The error message is : ' + str(e.message)
#!/usr/bin/env python
from datetime import datetime
from app import db
from BaseMixin import BaseMixin
from Log import Log, LogType
_log = Log()
class Reward(BaseMixin, db.Model):
__tablename__ = 'rewards'
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
name = db.Column(db.String(255), unique=True, nullable=False)
description = db.Column(db.String(255))
points = db.Column(db.Integer, nullable=False)
def __init__(self, name):
self.name = name
def __repr__(self):
return '<Reward %r>' % self.name
# Create operations
# Read operations
# Update operations
# Delete operations
def PostArticle(self,
user,
title,
content,
refile,
signature_id,
anony,
mailback,
session,
attach,
ignoreperm=False):
# check permission
if not ignoreperm:
self.PreparePostArticle(user, refile, anony, attach)
# filter title: 'Re: ' and '\ESC'
# title = title.replace('\033', ' ')
title = re.sub('[\x00-\x19]', ' ', title)
if (refile is None):
while (title[:4] == "Re: "):
title = title[4:]
if anony:
if not self.MayAnonyPost(user, refile):
anony = False
if attach:
if not self.CanPostAttach():
attach = None
post_file = PostEntry.PostEntry()
# Log.debug("PostArticle title: %s anony: %r" % (title, anony))
post_file.filename = self.GetPostFilename(False)
if (not anony):
post_file.owner = user.name
else:
post_file.owner = self.name
post_file.realowner = user.name
if (mailback):
post_file.SetMailBack(True)
content_encoded = Util.gbkEnc(content)
try:
with open(self.GetBoardPath() + post_file.filename, "ab") as f:
Post.Post.WriteHeader(f, user, False, self, title, anony, 0,
session)
f.write(content_encoded)
if (not anony):
Post.Post.AddSig(f, user, signature_id)
except IOError:
Log.error("PostArticle: write post failed!")
os.unlink(self.GetBoardPath() + post_file.filename)
raise ServerError("fail to write post file")
post_file.eff_size = len(content_encoded)
if (refile is not None):
post_file.rootcrc = refile.rootcrc
if (refile.IsRootPostAnonymous()):
post_file.SetRootPostAnonymous(True)
else:
mycrc = (~binascii.crc32(user.name, 0xffffffff)) & 0xffffffff
post_file.rootcrc = mycrc
if (anony):
post_file.SetRootPostAnonymous(True)
if (signature_id == 0):
has_sig = False
else:
has_sig = True
Post.Post.AddLogInfo(self.GetBoardPath(post_file.filename), user,
session, anony, has_sig)
post_file.title = Util.gbkEnc(title)
# TODO: outpost ('SS')
post_file.innflag = 'LL'
if attach:
try:
post_file.attachment = len(attach)
except:
post_file.attachment = 0
else:
post_file.attachment = 0
self.AfterPost(user, post_file, refile, anony)
if attach:
try:
for att in attach:
filename = att['name']
tmpfile = att['store_id']
if (not store.Store.verify_id(tmpfile)):
continue
tmpfile = store.Store.path_from_id(tmpfile)
Post.Post.AddAttach(self.GetBoardPath(post_file.filename),
filename, tmpfile)
except:
pass
if (not self.IsJunkBoard()):
user.AddNumPosts()
return True
from TwitchWebsocket import TwitchWebsocket
import json, requests, random, logging, time, re
from itertools import accumulate
from enum import Enum, auto
from Log import Log
Log(__file__)
from Settings import Settings
from Database import Database
class ResultCode(Enum):
SUCCESS = auto()
ERROR = auto()
class CommandType(Enum):
DEFINITION = 0
EXAMPLE = 1
class TwitchUrbanDictionary:
def __init__(self):
# Initialize variables
self.host = None
self.port = None
self.chan = None
self.nick = None
self.auth = None
self.max_chars = None
self.cooldown = None
self.banned_words = []
def __init__(self):
self.log = Log()
def modifyUser(self, uid):
Log.debug("modify user with uid " + uid)
jsonform = request.form['json']
return S3User.modify(uid, jsonform, self.getAdminConnection())
def EditPost(self,
session,
post_xid,
post_id=0,
new_title=None,
content=None,
mode='normal',
attach_to_remove=set(),
add_attach_list=[]):
(post_entry, post_id) = self.FindPost(post_id, post_xid, mode)
if post_entry is None:
raise NotFound("post not found")
if (self.name == "syssecurity" or self.name == "junk"
or self.name == "deleted"):
raise WrongArgs("can't edit post in board %s" % self.name)
if mode == "junk" or mode == "deleted":
raise WrongArgs("can't edit post in mode %s" % mode)
if self.CheckReadonly():
raise WrongArgs("board %s is read-only" % self.name)
user = session.GetUser()
if not post_entry.CanBeEdit(user, self):
raise NoPerm("you can't edit this post")
if self.DeniedUser(user):
raise NoPerm("you can't edit on board %s" % self.name)
post_path = self.GetBoardPath(post_entry.filename)
post = Post.Post(post_path, post_entry)
if content is None:
content = post.GetBody()
first_attach_pos = 0
need_update = False
new_post_path = post_path + ".new"
if new_title is not None and new_title != Util.gbkDec(
post_entry.title):
post_entry.title = Util.gbkEnc(new_title)
need_update = True
with open(post_path, "r+b") as postf:
Util.FLock(postf)
try:
attach_list = post.GetAttachList()
newpost = Post.Post(new_post_path, post_entry)
newpost.open()
try:
newpost.EditHeaderFrom(post, new_title)
size_header = newpost.pos()
newpost.EditContent(content, session, post)
content_len = newpost.pos() - size_header
if content_len != post_entry.eff_size:
post_entry.eff_size = content_len
need_update = True
# copy original attachments
orig_attach_id = 0
for attach_entry in attach_list:
if orig_attach_id not in attach_to_remove:
try:
attach_pos = newpost.AppendAttachFrom(
post, attach_entry)
if first_attach_pos == 0:
first_attach_pos = attach_pos
except:
pass
orig_attach_id += 1
# add new attachments
for attach_entry in add_attach_list:
filename = attach_entry['name']
tmpfile = attach_entry['store_id']
if (not store.Store.verify_id(tmpfile)):
continue
tmpfile = store.Store.path_from_id(tmpfile)
try:
attach_pos = newpost.AddAttachSelf(
filename, tmpfile)
if first_attach_pos == 0:
first_attach_pos = attach_pos
except Exception as e:
Log.warn("fail to add attach: %r" % e)
finally:
newpost.close()
os.rename(new_post_path, post_path)
finally:
try:
os.remove(new_post_path)
except:
pass
Util.FUnlock(postf)
if first_attach_pos != post_entry.attachment:
post_entry.attachment = first_attach_pos
need_update = True
if need_update:
# fail to update post info is not that important
if not self.UpdatePostEntry(post_entry, post_id, mode):
Log.warn("fail to update post entry!")
def getUser(self, uid):
Log.debug("get user with uid " + uid)
return S3User.view(uid, self.getAdminConnection())
def removeUser(self, uid):
Log.debug("remove user with uid " + uid)
return S3User.remove(uid, self.getAdminConnection())
class S3ObjectCtrl:
Log.debug("Entering in S3ObjectCtrl class <<")
def __init__(self, conf):
self.admin = conf.get("radosgw_admin", "admin")
self.key = conf.get("radosgw_key", "")
self.secret = conf.get("radosgw_secret", "")
self.conffile = conf.get("ceph_conf", '/etc/ceph/ceph.conf')
self.radosgw_url = conf.get("radosgw_url", "127.0.0.1")
self.clusterName = conf.get("cluster", "ceph")
if not self.radosgw_url.endswith('/'):
self.radosgw_url += '/'
self.url = self.radosgw_url + self.admin
self.cluster = rados.Rados(conffile=self.conffile)
print "\nlibrados version: " + str(self.cluster.version())
print "Will attempt to connect to: " + str(
self.cluster.conf_get('mon initial members'))
self.cluster.connect()
print "\nCluster ID: " + self.cluster.get_fsid()
#print "config url: "+self.url
#print "config admin: "+self.admin
#print "config key: "+self.key
#print(json.dumps(conf))
def getAdminConnection(self):
return S3Bucket(self.admin,
access_key=self.key,
secret_key=self.secret,
base_url=self.url)
def getObjectStructure(self):
startdate = datetime.now()
print(
str(startdate) + ' -Calling method getObjectStructure() begins <<')
print " __request", request
objectId = request.args.get('objectId')
bucketname = request.args.get('bucketName')
osd_dump = self.getOsdDump()
# objectIdd=request.form['objectId']
#bucketnamee=request.form['bucketName']
Log.debug("__getS3Object(objectId=" + str(objectId) +
", bucketName= " + str(bucketname) + ")")
#Log.debug("getS3Object(objectIdd="+str(objectIdd)+", bucketNamee= "+str(bucketnamee)+")")
#Retrieve the bucketId using the bucket name
bucketId = self.getBucketInfo(bucketname)["bucketid"]
#Get the pool name using the
poolname = self.getBucketInfo(bucketname)["poolname"]
#Retrieve the pool id
poolid = self.getPoolId(poolname)
#poolname=getPoolName(bucketName)
extended_objectId = bucketId + "_" + objectId
#Retrieve the user.rgw.manifest that contains the chunks list for the object
#usermnf=self.getUserRgwManifest(poolname,extended_objectId)
#Retrieve the chunk base name in the user.rgw.manifest attribute
chunkbasename = self.getChunkBaseName(poolname, extended_objectId)
print '__Chunk base name: ', chunkbasename
if len(chunkbasename): #chek if there is chunk por not for the object
#Retrieve the chunks list of the object
chunks = self.getChunks(bucketId, poolname, objectId,
chunkbasename)
chunks.append(extended_objectId
) #Add the last object that is around 512.0 kb
else:
chunks = [extended_objectId]
print "__Chunks list", chunks
#bucketInfo=self.getBucketInfo(bucketId)
chunklist = []
pgs = []
osds = []
osdids = []
pgid4osd = []
for chunk in chunks:
if len(chunk) > 0:
print 'Chunk= ', chunk
chunksize = self.getChunkSize(poolname, chunk)
pgid = self.getPgId(poolname, ' ' + chunk)
c = Chunk(chunk, chunksize, pgid[0])
chunklist.append(c)
if pgid4osd.count(pgid[1]) == 0:
pgid4osd.append(pgid[1])
if pgid4osd.count(pgid[0]) == 0:
pgid4osd.append(pgid[0])
#Create the PG for this chunk
#ef __init__(self,pgid,state,acting, up, acting_primary, up_primary):
pginfos = self.getOsdMapInfos(pgid[1])
pg = PG(pgid[0], pginfos['state'], pginfos['acting'],
pginfos['up'], pginfos['acting_primary'],
pginfos['up_primary'])
# print(pg.dump())
pgs.append(pg) #Append the PG in the pgs list
# print "____ OSD List for PG ", pgid[1],self.getOsdsListForPg(pgid[1])
for pgid in pgid4osd:
for id in self.getOsdsListForPg(
pgid): #sortir la boucle pour les pg
if osdids.count(id) == 0:
osdids.append(
id) #construct the list of the OSD to be displayed
#Log.debug("Total number of chunks retrived:"+str(nbchunks))
# print "_____osds list=",osdids
for osdid in osdids: #Loop the OSD list and retrieve the osd and add it in the osds list fot the S3 object
osd = self.getOsdInfos(osd_dump, osdid)
#print(osd.dump())
osds.append(osd)
s3object = S3Object(extended_objectId, bucketname, bucketId, poolid,
poolname, self.getPoolType(poolname, poolid),
self.getChunkSize(poolname, extended_objectId),
chunklist, pgs, osds)
print(s3object.dump())
duration = datetime.now() - startdate
Log.info(
str(datetime.now()) +
' ___Calling method getObjectStructure() end >> duration= ' +
str(duration.seconds))
return s3object.dump()
#This method returns the pool id of a given pool name
def getPoolId(self, poolname):
Log.info("___getPoolId(poolname=" + str(poolname) + ")")
outdata = self.executeCmd('ceph osd pool stats ', [poolname], [])
poolid = outdata.strip().split('\n')[0].split(' id ')[
1] #['pool .rgw.buckets', ' 16']
return poolid
# This method returns the pool type of a pool using the poolname and the pool id parameters
def getPoolType(self, poolname, poolId):
Log.info("___getPoolType(poolname=" + str(poolname) + ", poolId=" +
str(poolId) + ")")
outdata = self.executeCmd('ceph osd dump ', [],
[poolname, ' ' + poolId])
pooltype = outdata.strip().split(
' '
)[3] #['pool', '26', "'.rgw.buckets'", 'replicated', 'size', '2', 'min_size', '1', 'crush_ruleset', '0', 'object_hash', 'rjenkins', 'pg_num', '8', 'pgp_num', '8', 'last_change', '408', 'stripe_width', '0']
return pooltype
# This method computes the size of an object
# arguments: bucketName: The bucket name to look for
# objectId: the object id we want to compute the size
def getChunkSize(self, poolname, objectid): # Method OK
Log.info('___getChunkSize(poolName=' + str(poolname) + ', objectId=' +
str(objectid) + ')')
ioctx = self.cluster.open_ioctx(poolname)
size = ioctx.stat(str(objectid))
return int(size[0])
#This method returns the lists of osds for a given pgid
# The following command is performed : ceph pg map 16.7 result= osdmap e11978 pg 16.7 (16.7) -> up [9,6] acting [9,6]
def getUpActing(self, pgid):
Log.info("___getUpActing(pgid=" + str(pgid) + ")")
outdata = self.executeCmd('ceph pg map ', [pgid], [])
pgid = outdata.strip().split(' -> ', 2)[1].split(
' ', 4) #'up' '[9,6]' 'acting' '[9,6]'
osds = {"up": pgid[1], "acting": pgid[3]}
return osds
# This method retrieves the information about the status of an osd: acting, up, primary_acting, primary_up
# The PG id is used as an input argument
def getOsdMapInfos(self, pgid):
Log.info("___getOsdMapInfos(pgid=" + str(pgid) + ")")
cephRestApiUrl = getCephRestApiUrl(
request) + 'tell/' + pgid + '/query.json'
Log.debug("____cephRestApiUrl Request=" + cephRestApiUrl)
osdmap = []
data = requests.get(cephRestApiUrl)
r = data.content
if data.status_code != 200:
print 'Error ' + str(
data.status_code) + ' on the request getting pools'
return osdmap
#print(r)
if len(r) > 0:
osdmap = json.loads(r)
else:
Log.err('The getOsdMapInfos() method returns empty data')
osdmap = json.loads(r)
#osdmap=r.read()
# Log.debug(osdmap)
acting = osdmap["output"]["acting"]
up = osdmap["output"]["up"]
state = osdmap["output"]["state"]
acting_primary = osdmap["output"]["info"]["stats"]["acting_primary"]
up_primary = osdmap["output"]["info"]["stats"]["up_primary"]
osdmap_infos = {
"acting": acting,
"acting_primary": acting_primary,
"state": state,
"up": up,
"up_primary": up_primary
}
return osdmap_infos
#{
# id : "osd.1",
# status : ['in','up'],
# host : "p-sbceph12",
# capacity : 1000000000,
# occupation : 0.236
# },
#This method returns the information for a given osd id that is passed in argument
#The information of the osd is retrieved thanks to mongoDB inkscopeCtrl/{clusterName}/osd?depth=2 REST URI
def getOsdDump(self):
Log.debug("___getOsdDump()")
#print str(datetime.datetime.now()), "-- Process OSDDump"
cephRestUrl = request.url_root + self.clusterName + '/osd?depth=2'
print(cephRestUrl)
# Set HTTP credentials for url callback (requests.)
data = requests.get(cephRestUrl)
#
osds = []
if data.status_code != 200:
print 'Error ' + str(
data.status_code) + ' on the request getting osd'
return osds
r = data.content
if len(r) > 0:
osds = json.loads(r)
else:
Log.err('The osd dump returns empty data')
return osds
def getOsdInfos(self, osds, osdid):
Log.info("___getOsdInfos(osdid=" + str(osdid) + ")")
i = 0
osdnodename = ''
capacity = 0
used = 0
total = 1
hostid = ''
while i < len(osds):
if osds[i]["node"]["_id"] == osdid:
osdnodename = osds[i]["node"]["name"]
break
i = i + 1
Log.debug("_____OSD Node Name= " + str(osdnodename))
stat = []
try:
while i < len(osds):
if osds[i]["stat"]["osd"]["node"]["$id"] == osdid:
state = osds[i]["stat"]
if state["in"]:
stat.append("in")
if state["up"]:
stat.append("up")
hostid = osds[i]["stat"]["osd"]["host"]["$id"]
capacity = osds[i]["host"]["network_interfaces"][0][
"capacity"]
if osds[i]["partition"]["stat"] != 'null':
used = osds[i]["partition"]["stat"]["used"]
if osds[i]["partition"]["stat"] != 'null':
total = osds[i]["partition"]["stat"]["total"]
break
i = i + 1
except TypeError, e:
Log.err(e.__str__())
#Log.debug("OSD node infos [: ")
print "|_______________ [up, acting ]=", stat
Log.debug("|______________Host id=" + str(hostid))
Log.debug("|______________Capacity =" + str(capacity))
Log.debug("|______________Used =" + str(used))
Log.debug("|______________Total =" + str(total))
Log.debug(" ]")
occupation = "null"
if int(used) > 0:
occupation = round(float(used) / float(total), 3)
osd = OSD(osdnodename, stat, hostid, capacity, occupation)
print "_______________ osd= ", osd.dump()
return osd
def add_object(self):
log = Log(self.display,0,0,self.height)
return log
