def loadReservations(userJsonFn, reservationsJsonFn, fromDate, toDate,
fetchData=True):
""" Load reservation list either from the booking system or from a
local json file with cached data.
Params:
userJsonFn: user credentials to login into the booking system
reservationsJsonFile: local file to use in case the reservations can
not be load from the booking system.
fromDate-toDate: date range to retrieve reservations
"""
if fetchData:
try:
bMan = BookingManager()
print("Loading reservations from booking system...")
t = pwutis.Timer()
t.tic()
reservationsJson = bMan.fetchReservationsJson(userJsonFn,
fromDate, toDate)
if reservationsJson is None:
raise Exception("Could not fetch data from the booking system.")
t.toc()
with open(reservationsJsonFn, 'w') as reservationsFile:
json.dump(reservationsJson, reservationsFile)
except Exception as ex:
print("Error: ", ex)
print("Trying to load reservations from file:", reservationsJsonFn)
jsonFile = open(reservationsJsonFn)
reservationsJson = json.load(jsonFile)
jsonFile.close()
return loadReservationsFromJson(reservationsJson)
def _runNewItem(self, itemType, itemName):
if self._match(itemName):
spaces = (itemType * 2) * ' '
scipion = join(os.environ['SCIPION_HOME'], 'scipion')
if (itemName.startswith("xmipp_")):
cmd = "%s %s %s" % (spaces, scipion, itemName)
else:
cmd = "%s %s test %s" % (spaces, scipion, itemName)
run = ((itemType == MODULE and self.mode == 'module')
or (itemType == CLASS and self.mode == 'classes')
or (itemType == TEST and self.mode == 'all'))
if run:
if self.log:
logFile = join(self.testsDir, '%s.txt' % itemName)
cmdFull = cmd + " > %s 2>&1" % logFile
else:
logFile = ''
cmdFull = cmd
print pwutils.green(cmdFull)
t = pwutils.Timer()
t.tic()
self.testCount += 1
result = os.system(cmdFull)
if self.log:
self._logTest(cmd.replace(scipion, 'scipion'), t.getToc(),
result, logFile)
def runTests(self, tests):
self.testCount = 0
if self.log:
self.testsDir = join(os.environ['SCIPION_USER_DATA'], 'Tests',
self.log)
pwutils.cleanPath(self.testsDir)
pwutils.makePath(self.testsDir)
self.testLog = join(self.testsDir, 'tests.html')
self.testTimer = pwutils.Timer()
self.testTimer.tic()
self.headerPrefix = '<h3>Test results (%s) Duration: ' % pwutils.prettyTime(
)
f = open(self.testLog, 'w')
f.write("""<!DOCTYPE html>
<html>
<body>
""")
f.write(self.headerPrefix + '</h3>')
f.write("""
<table style="width:100%" border="1">
<tr>
<th>#</th>
<th>Command</th>
<th>Time</th>
<th>Result</th>
<th>Log file</th>
</tr>
<!-- LAST_ROW -->
</table>
</body>
</html>""")
f.close()
self._visitTests(tests, self._runNewItem)
if self.log:
print "\n\nOpen results in your browser: \nfile:///%s" % self.testLog
type=str,
default='cem',
dest='sort',
metavar='SORT',
help="Sort criteria. (cem, date)")
_addArg("--filter",
type=str,
default='',
dest='filter',
metavar='SORT',
help="Filter orders base on some criteria. (bag)")
args = parser.parse_args()
t = pwutils.Timer()
t.tic()
apiJsonFile = 'data/%s' % PORTAL_API
pMan = PortalManager(apiJsonFile)
def writeOrders(ordersJson, fn):
with open(fn, 'w') as ordersFile:
print("Writing orders JSON to file: ", fn)
json.dump(ordersJson, ordersFile, indent=2)
if not args.detailsCem:
def test_mapper(self):
""" test that indexes are created when a
setOfCoordinates is created """
PARTNUMBER = 10
MICNUMBER = 600
NUMBERCOORDINATES = PARTNUMBER * MICNUMBER
indexesNames = ['_micId']
prot = createDummyProtocol("dummy_protocol")
# create set of micrographs
micSet = emobj.SetOfMicrographs(filename=":memory:")
mic = emobj.Micrograph()
for i in range(NUMBERCOORDINATES):
mic.setLocation(i, "mic_%06d.mrc" % i)
micSet.append(mic)
mic.cleanObjId()
micSet.write()
# create a set of particles
coordSet = prot._createSetOfCoordinates(micSet)
coord = emobj.Coordinate()
for i in range(NUMBERCOORDINATES):
coordSet.append(coord)
coord.cleanObjId()
coordSet.write()
# check defined indexes
setOfCoordinatesFileName = \
prot._getPath("coordinates.sqlite")
print(os.path.abspath(setOfCoordinatesFileName))
indexes = sorted(
[index[1] for index in getIndex(setOfCoordinatesFileName)])
for index, indexName in zip(indexes, indexesNames):
self.assertEqual(index, 'index_' + indexName)
# Test speed: based on loop in file protocol_extractparticles.py
# for 600 mic and 100 part the values for the first
# second and third case where:
# Loop with index: 5 sec
# Loop no index: 8:01 min
# Loop optimized code: 4 sec
# for 6000 mic and 200 part the values for the first
# Loop with index: 1:47 min
# optimized Loop with index: 1:20 min
# Loop no index: after several hours I stopped the process
SPEEDTEST = True
if SPEEDTEST: # code from protocol_particles. line 415
testTimer = pwutils.Timer()
testTimer.tic()
for mic in micSet:
micId = mic.getObjId()
coordList = []
for coord in coordSet.iterItems(where='_micId=%s' % micId):
coordList.append(coord.clone())
testTimer.toc("Loop with INDEX took:")
lastMicId = None
testTimer.tic()
for coord in coordSet.iterItems(orderBy='_micId', direction='ASC'):
micId = coord.getMicId()
if micId != lastMicId:
lastMicId = micId
coordList = []
coordList.append(coord.clone())
testTimer.toc("Loop with INDEX and proper code, took:")
# delete INDEX, this will not work
# if database is not sqlite
conn = sqlite3.connect(setOfCoordinatesFileName)
cur = conn.cursor()
for index in indexesNames:
cur.execute("DROP INDEX index_%s" % index)
cur.close()
conn.close()
testTimer.tic()
for mic in micSet:
micId = mic.getObjId()
coordList = []
for coord in coordSet.iterItems(where='_micId=%s' % micId):
coordList.append(coord.clone())
testTimer.toc("Loop with NO INDEX took:")
lastMicId = None
testTimer.tic()
for coord in coordSet.iterItems(orderBy='_micId', direction='ASC'):
micId = coord.getMicId()
if micId != lastMicId:
lastMicId = micId
coordList = []
coordList.append(coord.clone())
testTimer.toc("Loop with NO INDEX but proper code, took:")