def getSizeTimeRate(logarr):
"""
Extract the size,rate and time from an array of ARCHIVE log entries
related to a single ARCHIVE request.
"""
saveLine = filter(lambda x: x.find('Saved data in file') > -1, logarr)[0]
splitLine = saveLine.split()
(size, time, rate) = (float(splitLine[-8]), float(splitLine[-6]),
float(splitLine[-3]))
tend = DateTime.DateTimeFrom(splitLine[0]) # end transfer
saveLine = filter(lambda x: x.find('Saving data in file') > -1, logarr)[0]
tstart = DateTime.DateTimeFrom(saveLine.split()[0]) # start of transfer
return (size, time, rate, tstart, tend)
def __init__(self, institution, subject, year, timecode, typecode, version,
datetime, place, access):
""" Create a new exam object. """
# Check the argument types, and throw error if not as expected
for (arg, typ) in (
('institution', int),
('year', int),
('version', (str, unicode)),
('subject', (str, unicode)),
('timecode', (str, unicode)),
('typecode', (str, unicode)),
('place', (str, unicode)),
('access', (str, unicode)),
('datetime', (str, unicode, DateTime)),
):
val = locals().get(arg)
assert isinstance(
val,
typ), "%s must be %s, was %s" % (arg, repr(typ), type(val))
self.institution = int(institution)
self.year = int(year)
self.version = unicode(version)
self.subject = unicode(subject)
self.timecode = unicode(timecode)
self.typecode = unicode(typecode)
self.place = unicode(place)
self.access = unicode(access) # FIXME if anything changes.
self.datetime = DateTime.DateTimeFrom(datetime)
self.candidates = set()
def parseLog(self, log=''):
rexp = re.compile('[\[\]]')
(logTime, logType, logMsg, logTrace) = rexp.split(log, 3)
if logType not in self.LogTypes:
raise exceptions.Exception, "unknown log type %s" % logType
# lT = DateTime.ISO.ParseDateTime(logTime.strip())
return (DateTime.DateTimeFrom(logTime.strip()), logType.strip(), logMsg.strip(), \
logTrace.strip()[:-1])
def test(api):
avZone = 'us-east-1e'
date = DateTime.DateTimeFrom('2018-05-01')
dateStr = date.Format('%Y%m%d')
volume = createVolume(api, date, avZone)
instance = runInstance(api, date, avZone, volume)
api.runTask('overseer',
taskset='gu-%s' % dateStr,
instance=instance,
wait=True)
api.runTask('jetdb',
taskset='gu-%s' % dateStr,
instance=instance,
wait=True)
def getLatestStoryLinks(limit):
sql = """
select s.id, g.title, s.published from stories s, games g
where s.game=g.id and g.status='published'
order by s.published desc limit %s
""" % limit
c = getDBI().conn.cursor()
c.execute(sql)
rows = c.fetchall()
# manually convert the date to a datetime.
# we could also use PyDO.postconn._dateConvertFromDB
# instead of DateTimeFrom.
return [[x, y, M.DateTimeFrom(z)] for x, y, z in rows]
def _old_test():
print "testing old functions..."
print timeConv('1h20m3s')
print time.asctime(time.gmtime(datestrToSeconds('10:20:03')))
print time.asctime(time.gmtime(datestrToSeconds('2/11/1999 10:20:03')))
print currentDate()
print "testing new functions..."
cur = DateTime.now()
print cur.ticks(), cur
for test in [
':15',
(':15', ':30', ':45', ':00', '06:00'),
'00:15',
'23:59',
'23:59:59',
'2007-05-06',
'2007-05-06 23:59',
'2007-05-06 23:59:59',
cur + 0.5,
DateTime.RelativeDateTime(minute=15),
DateTime.RelativeDateTime(minute=00),
]:
scs = None
try:
scs = convertUntil(test)
status = 'good'
except PastTimeException:
status = 'time is past'
except TimeException:
status = 'bad argument'
print test
print '-->', scs,
if scs is not None:
print DateTime.DateTimeFrom(scs),
else:
print "None",
print status
def run_query():
cur_buy.execute(
"SELECT bid,current_market.systemid,current_market.stationid,price,volremain,(issued+duration),range,regionname, (reportedtime),stationname,security,minvolume,regions.regionid,orderid FROM current_market,regions,stations,systems WHERE "
+ reg_block +
" AND stations.systemid = systems.systemid AND typeid = %s AND stations.stationid = current_market.stationid AND current_market.regionid = regions.regionid AND now() - reportedtime <= '"
+ sql_age +
"' AND volremain >= %s AND current_market.bid = 1 " +
sql_system + " ORDER BY " + order + " " + borderdir + " " +
limit, [typeid, minQ])
cur_sell.execute(
"SELECT bid,current_market.systemid,current_market.stationid,price,volremain,(issued+duration),range,regionname,(reportedtime),stationname,security,regions.regionid,orderid FROM current_market,regions,stations,systems WHERE "
+ reg_block +
" AND typeid = %s AND stations.systemid = systems.systemid AND stations.stationid = current_market.stationid AND current_market.regionid = regions.regionid AND now() - reportedtime <= '"
+ sql_age +
"' AND volremain >= %s AND current_market.bid = 0 " +
sql_system + " ORDER BY " + order + " " + orderdir + " " +
limit, [typeid, minQ])
for (query, lista, isbuy) in [(cur_buy, buys, True),
(cur_sell, sells, False)]:
r = query.fetchone()
while r:
rec = {}
rec['systemid'] = r[1]
rec['stationid'] = r[2]
price = float(r[3])
string = format_price(price)
rec['price'] = string
rec['price_raw'] = price
rec['volremain'] = format_long(r[4])
rec['volremain_raw'] = r[4]
rec['expires'] = str(r[5])[0:10]
if r[0] == True:
rec['range'] = r[6]
else:
rec['range'] = -2
rec['regionname'] = r[7]
reported = DateTime.DateTimeFrom(str(r[8]))
now = DateTime.gmt()
hours = (now - reported).hours
if hours >= 1:
rec['reportedtime'] = "%d hours ago" % (hours)
else:
rec['reportedtime'] = "%d minutes ago" % (
now - reported).minutes
rec['stationname'] = r[9]
sec = r[10]
ssec = str(sec)[0:3]
if sec <= 0.0:
ssec = "0.0"
rec['security'] = ssec
# Try to grab regionid from the end of the query
if isbuy:
if int(r[11]) > 1:
rec['minvolume'] = format_long(int(r[11]))
rec['minvolume_raw'] = int(r[11])
else:
rec['minvolume'] = 1
rec['minvolume_raw'] = 1
rec['regionid'] = r[12]
rec['orderid'] = r[13]
else:
rec['minvolume'] = 1
rec['minvolume_raw'] = 1
rec['regionid'] = r[11]
rec['orderid'] = r[12]
lista.append(rec)
r = query.fetchone()
"""
s3 = S3(api)
userList = s3.list_dir('obsidian-cluster-share', 'griffith/exports/')
userList.update(
s3.list_dir('obsidian-cluster-share', 'griffith/exports_v2/'))
userList.discard('00-Global')
print "Got %d users with export data." % len(userList)
with open('all_usernames_in_export.txt', 'w') as fh:
userListV2 = list(userList)
userListV2.sort()
for un in userListV2:
fh.write('%s\n' % un)
print "..done"
if __name__ == "__main__":
import sys
clusterCfg = cluster.Cluster('griffith.yaml', {
'realm': 'griffith',
})
api = aws.AWS(cluster=clusterCfg)
api.cluster.args.update(getFrank())
args = sys.argv[1:]
startDate = endDate = None
if args:
startDate = DateTime.DateTimeFrom(args.pop(0))
if args:
endDate = DateTime.DateTimeFrom(args.pop(0))
if startDate:
process(api, startDate, endDate)
