def test_remove_session(self):
b = Bank('alu')
for i in range(1,5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
self.assertTrue(len(b.bank['sessions'])==4)
b.remove_session(b.session.get('id'))
self.assertTrue(len(b.bank['sessions'])==3)
def test_remove_session(self):
b = Bank('alu')
for i in range(1, 5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
self.assertTrue(len(b.bank['sessions']) == 4)
b.remove_session(b.session.get('id'))
self.assertTrue(len(b.bank['sessions']) == 3)
def test_remove_session(self):
b = Bank("alu")
for i in range(1, 5):
s = Session("alu", self.config, UpdateWorkflow.FLOW)
s._session["status"][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
self.assertTrue(len(b.bank["sessions"]) == 4)
b.remove_session(b.session.get("id"))
self.assertTrue(len(b.bank["sessions"]) == 3)
def test_clean_old_sessions(self):
'''
Checks a session is used if present
'''
b = Bank('local')
for i in range(1, 5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
b2 = Bank('local')
b2.update()
b2.clean_old_sessions()
self.assertTrue(len(b2.bank['sessions']) == 1)
def test_session_reload_notover(self):
'''
Checks a session is used if present
'''
b = Bank('alu')
for i in range(1, 5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
self.assertTrue(b.session.get_status(Workflow.FLOW_INIT))
def test_clean_old_sessions(self):
'''
Checks a session is used if present
'''
b = Bank('local')
for i in range(1,5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
b2 = Bank('local')
b2.update()
b2.clean_old_sessions()
self.assertTrue(len(b2.bank['sessions']) == 1)
def test_session_reload_notover(self):
'''
Checks a session is used if present
'''
b = Bank('alu')
for i in range(1,5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
self.assertTrue(b.session.get_status(Workflow.FLOW_INIT))
def test_clean_old_sessions(self):
"""
Checks a session is used if present
"""
b = Bank("local")
for i in range(1, 5):
s = Session("alu", self.config, UpdateWorkflow.FLOW)
s._session["status"][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
b2 = Bank("local")
b2.update()
b2.clean_old_sessions()
self.assertTrue(len(b2.bank["sessions"]) == 1)
def test_session_reload_notover(self):
"""
Checks a session is used if present
"""
b = Bank("alu")
for i in range(1, 5):
s = Session("alu", self.config, UpdateWorkflow.FLOW)
s._session["status"][Workflow.FLOW_INIT] = True
b.session = s
b.save_session()
b = Bank("alu")
b.load_session(UpdateWorkflow.FLOW)
self.assertTrue(b.session.get_status(Workflow.FLOW_INIT))
def test_session_reload_over(self):
"""
Checks a session if is not over
"""
b = Bank('alu')
for i in range(1, 5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
s._session['status'][Workflow.FLOW_OVER] = True
b.session = s
b.save_session()
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
self.assertFalse(b.session.get_status(Workflow.FLOW_INIT))
def test_session_reload_over(self):
"""
Checks a session if is not over
"""
b = Bank('alu')
for i in range(1,5):
s = Session('alu', self.config, UpdateWorkflow.FLOW)
s._session['status'][Workflow.FLOW_INIT] = True
s._session['status'][Workflow.FLOW_OVER] = True
b.session = s
b.save_session()
b = Bank('alu')
b.load_session(UpdateWorkflow.FLOW)
self.assertFalse(b.session.get_status(Workflow.FLOW_INIT))
def migrate_bank(cur, bank, history=False):
"""
Migrate old MySQL information to the new MongoDB
:param cur: MySQL cursor
:type cur: MySQL cursor
:param bank: Bank name
:type bank: String
:param history: Keep bank history
:type history: Boolean, default False
:return:
"""
query = "SELECT p.path, p.session, p.creation, p.remove, p.size, u.updateRelease, s.logfile from productionDirectory p "
query += "JOIN updateBank u on u.idLastSession = p.session JOIN bank b on b.idbank = p.ref_idbank "
query += "LEFT JOIN session s ON s.idsession = u.idLastSession "
query += "WHERE b.name='" + str(bank) + "' "
if not history:
query += "AND p.remove IS NULL "
query += "ORDER BY p.creation ASC"
cur.execute(query)
banks = []
not_prod = {}
for row in cur.fetchall():
banks.append({
'path': row[0],
'session': row[1],
'creation': row[2],
'remove': row[3],
'size': row[4],
'release': row[5],
'remoterelease': row[5],
'logfile': row[6]
})
# If we want to keep the history we will need to delete from 'production',
# session(s) which have been tagged as 'removed', so row[3] is a date
if row[3] and history:
sess = time.mktime(
datetime.datetime.strptime(str(row[2]),
"%Y-%m-%d %H:%M:%S").timetuple())
not_prod[sess] = True
for prod in banks:
pathelts = prod['path'].split('/')
release_dir = pathelts[len(pathelts) - 1]
prod['prod_dir'] = release_dir
pattern = re.compile(".*__(\d+)$")
relmatch = pattern.match(release_dir)
if relmatch:
prod['release'] = prod['release'] + '__' + relmatch.group(1)
# We create the session id from the productionDirectory.creation field
session_id = time.mktime(
datetime.datetime.strptime(str(prod['creation']),
"%Y-%m-%d %H:%M:%S").timetuple())
session_exists = False
b = Bank(bank, no_log=True)
# We check we did not already imported this session into the database
for s in b.bank['sessions']:
if s['id'] == session_id:
logging.warn('Session already imported: ' + b.name + ':' +
str(prod['creation']))
session_exists = True
break # No need to continue
if session_exists:
continue
for p in b.bank['production']:
if p['release'] == prod['release']:
logging.warn('Prod release already imported: ' + b.name + ":" +
p['release'])
continue
b.load_session(UpdateWorkflow.FLOW)
b.session.set('prod_dir', prod['prod_dir'])
b.session.set('action', 'update')
b.session.set('release', prod['release'])
b.session.set('remoterelease', prod['remoterelease'])
# Set production size from productionDirectory.size field
b.session.set('fullsize', prod['size'])
b.session._session['status'][Workflow.FLOW_OVER] = True
b.session._session['update'] = True
# We set the session.id (timestamp) with creation field from productionDirectory table
b.session.set('id', session_id)
b.save_session()
# Keep trace of the logfile. We need to do a manual update
if prod['logfile'] and os.path.exists(prod['logfile']):
b.banks.update({
'name': b.name,
'sessions.id': session_id
}, {'$set': {
"sessions.$.log_file": prod['logfile']
}})
# Due to the way save_session set also the production, to exclude last session
# from the production entries, we need to loop over each production entries
if history:
for production in b.bank['production']:
if production['session'] in not_prod:
b.banks.update(
{
'name': b.name,
'production.session': production['session']
}, {
'$pull': {
'production': {
'session': production['session']
}
}
})
# Listing files ?
root_files = []
if os.path.exists(prod['path']):
root_files = os.listdir(prod['path'])
for root_file in root_files:
if root_file.startswith('listing.'):
fileName, fileExtension = os.path.splitext(root_file)
f = open(os.path.join(prod['path'], root_file), 'r')
listing = f.read()
f.close()
f = open(
os.path.join(prod['path'], 'listingv1' + fileExtension),
'w')
listing = "{\"files\": [], \"name\": \"" + fileExtension.replace(
'.', '') + "\"," + listing + "}"
f.write(listing)
f.close()
# Current link?
pathelts = prod['path'].split('/')
del pathelts[-1]
current_link = os.path.join('/'.join(pathelts), 'current')
if os.path.lexists(current_link):
b.bank['current'] = b.session._session['id']
b.banks.update({'name': b.name},
{'$set': {
'current': b.session._session['id']
}})
def migrate_bank(cur, bank, history=False):
"""
Migrate old MySQL information to the new MongoDB
:param cur: MySQL cursor
:type cur: MySQL cursor
:param bank: Bank name
:type bank: String
:param history: Keep bank history
:type history: Boolean, default False
:return:
"""
query = "SELECT p.path, p.session, p.creation, p.remove, p.size, u.updateRelease, s.logfile, s.status, r.protocol, "
query += "r.server, r.remoteDir FROM productionDirectory p "
query += "JOIN updateBank u on u.idLastSession = p.session JOIN bank b on b.idbank = p.ref_idbank "
query += "LEFT JOIN session s ON s.idsession = u.idLastSession "
query += "JOIN configuration c ON c.idconfiguration = u.ref_idconfiguration JOIN remoteInfo r "
query += "ON r.idremoteInfo = c.ref_idremoteInfo "
query += "WHERE b.name='" + str(bank) + "' "
if not history:
query += "AND p.remove IS NULL "
query += "ORDER BY p.creation ASC"
cur.execute(query)
banks = []
not_prod = {}
for row in cur.fetchall():
banks.append({
'path': row[0],
'session': row[1],
'creation': row[2],
'remove': row[3],
'size': humanfriendly.parse_size(row[4].replace(',', '.')),
'release': row[5],
'remoterelease': row[5],
'logfile': row[6],
'status': row[7],
'protocol': row[8],
'server': row[9],
'remotedir': row[10]
})
# If we want to keep the history we will need to delete from 'production',
# session(s) which have been tagged as 'removed', so row[3] is a date
if row[3] and history:
sess = time.mktime(
datetime.datetime.strptime(str(row[2]),
"%Y-%m-%d %H:%M:%S").timetuple())
not_prod[sess] = True
for prod in banks:
pathelts = prod['path'].split('/')
release_dir = pathelts[len(pathelts) - 1]
prod['prod_dir'] = release_dir
pattern = re.compile(".*__(\d+)$")
relmatch = pattern.match(release_dir)
if relmatch:
prod['release'] = prod['release'] + '__' + relmatch.group(1)
# We create the session id from the productionDirectory.creation field
session_id = time.mktime(
datetime.datetime.strptime(str(prod['creation']),
"%Y-%m-%d %H:%M:%S").timetuple())
session_exists = False
b = Bank(bank, no_log=True)
# We check we did not already imported this session into the database
for s in b.bank['sessions']:
if s['id'] == session_id:
logging.warn('Session already imported: ' + b.name + ':' +
str(prod['creation']))
session_exists = True
break # No need to continue
if session_exists:
continue
for p in b.bank['production']:
if p['release'] == prod['release']:
logging.warn('Prod release already imported: ' + b.name + ":" +
p['release'])
continue
b.load_session(UpdateWorkflow.FLOW)
b.session.set('prod_dir', prod['prod_dir'])
b.session.set('action', 'update')
b.session.set('release', prod['release'])
b.session.set('remoterelease', prod['remoterelease'])
# Biomaj >= 3.0.14 introduce new field in sessions 'workflow_status'
# Set production size from productionDirectory.size field
b.session.set('workflow_status', True if prod['status'] else False)
b.session.set('fullsize', prod['size'])
b.session._session['status'][Workflow.FLOW_OVER] = True
b.session._session['update'] = True
# We set the session.id (timestamp) with creation field from productionDirectory table
b.session.set('id', session_id)
b.save_session()
# We need set update the field 'last_update_time' to the time the bank has been created
# because 'save_session' set this value to the time it is called
b.banks.update({
'name': b.name,
'sessions.id': session_id
}, {'$set': {
'sessions.$.last_update_time': session_id
}})
# Keep trace of the logfile. We need to do a manual update
if prod['logfile'] and os.path.exists(prod['logfile']):
b.banks.update({
'name': b.name,
'sessions.id': session_id
}, {'$set': {
'sessions.$.log_file': prod['logfile']
}})
# Downloaded and local files (created by 'bank.save_session')
# remove as created empty due to 'save_sessions' call
cache_dir = b.config.get('cache.dir')
download_file = os.path.join(cache_dir, 'files_' + str(session_id))
local_file = os.path.join(cache_dir, 'local_files_' + str(session_id))
if os.path.isfile(download_file):
os.unlink(download_file)
if os.path.isfile(local_file):
os.unlink(local_file)
# Due to the way save_session set also the production, to exclude last session
# from the production entries, we need to loop over each production entries
if history:
# If we want to keep history, we also need to keep trace of the time session/update has been deleted
# from the database/disk
if prod['remove']:
removed = time.mktime(
datetime.datetime.strptime(str(
prod['remove']), "%Y-%m-%d %H:%M:%S").timetuple())
b.banks.update({
'name': b.name,
'sessions.id': session_id
}, {'$set': {
'sessions.$.deleted': removed
}})
for production in b.bank['production']:
if production['session'] in not_prod:
b.banks.update(
{
'name': b.name,
'production.session': production['session']
}, {
'$pull': {
'production': {
'session': production['session']
}
}
})
# Listing files ?
root_files = []
if os.path.exists(prod['path']):
root_files = os.listdir(prod['path'])
for root_file in root_files:
if root_file.startswith('listing.'):
fileName, fileExtension = os.path.splitext(root_file)
f = open(os.path.join(prod['path'], root_file), 'r')
listing = f.read()
f.close()
f = open(
os.path.join(prod['path'], 'listingv1' + fileExtension),
'w')
listing = "{\"files\": [], \"name\": \"" + fileExtension.replace(
'.', '') + "\"," + listing + "}"
f.write(listing)
f.close()
if root_file.startswith('flat') and session_id not in not_prod:
flat_dir = os.path.join(prod['path'], root_file)
if not os.path.exists(flat_dir):
print(
"[%s] [WARN] Can't list %s/flat directory. Skipping ..."
% (b.name, prod['path']))
continue
flat_files = os.listdir(flat_dir)
files_info = []
for flat_file in flat_files:
try:
file_info = os.stat(os.path.join(flat_dir, flat_file))
except IOError as err:
print(
"[%s][WARN] Failed to stat get info from %s: %s" %
(b.name, str(file_info), str(err)))
continue
ctime = datetime.datetime.fromtimestamp(file_info[9])
files_info.append({
'name':
flat_file,
'save_as':
flat_file,
'user':
file_info[4],
'group':
file_info[5],
'year':
ctime.year,
'day':
ctime.day,
'month':
ctime.month,
'root':
prod['remotedir'],
'url':
prod['protocol'] + '://' + prod['server'],
'size':
file_info[6]
})
if len(files_info) > 0:
# Saved files
local_file = open(local_file, 'w')
local_file.write(json.dumps(files_info))
local_file.close()
# Current link?
pathelts = prod['path'].split('/')
del pathelts[-1]
current_link = os.path.join('/'.join(pathelts), 'current')
if os.path.lexists(current_link):
b.bank['current'] = b.session._session['id']
b.banks.update({'name': b.name},
{'$set': {
'current': b.session._session['id']
}})
