def test_remove_session(self): b = Bank('alu') for i in range(1,5): s = Session('alu', self.config, UpdateWorkflow.FLOW) s._session['status'][Workflow.FLOW_INIT] = True b.session = s b.save_session() self.assertTrue(len(b.bank['sessions'])==4) b.remove_session(b.session.get('id')) self.assertTrue(len(b.bank['sessions'])==3)
def test_remove_session(self): b = Bank('alu') for i in range(1, 5): s = Session('alu', self.config, UpdateWorkflow.FLOW) s._session['status'][Workflow.FLOW_INIT] = True b.session = s b.save_session() self.assertTrue(len(b.bank['sessions']) == 4) b.remove_session(b.session.get('id')) self.assertTrue(len(b.bank['sessions']) == 3)
def test_remove_session(self): b = Bank("alu") for i in range(1, 5): s = Session("alu", self.config, UpdateWorkflow.FLOW) s._session["status"][Workflow.FLOW_INIT] = True b.session = s b.save_session() self.assertTrue(len(b.bank["sessions"]) == 4) b.remove_session(b.session.get("id")) self.assertTrue(len(b.bank["sessions"]) == 3)
def test_clean_old_sessions(self): ''' Checks a session is used if present ''' b = Bank('local') for i in range(1, 5): s = Session('alu', self.config, UpdateWorkflow.FLOW) s._session['status'][Workflow.FLOW_INIT] = True b.session = s b.save_session() b2 = Bank('local') b2.update() b2.clean_old_sessions() self.assertTrue(len(b2.bank['sessions']) == 1)
def test_session_reload_notover(self): ''' Checks a session is used if present ''' b = Bank('alu') for i in range(1, 5): s = Session('alu', self.config, UpdateWorkflow.FLOW) s._session['status'][Workflow.FLOW_INIT] = True b.session = s b.save_session() b = Bank('alu') b.load_session(UpdateWorkflow.FLOW) self.assertTrue(b.session.get_status(Workflow.FLOW_INIT))
def test_clean_old_sessions(self): ''' Checks a session is used if present ''' b = Bank('local') for i in range(1,5): s = Session('alu', self.config, UpdateWorkflow.FLOW) s._session['status'][Workflow.FLOW_INIT] = True b.session = s b.save_session() b2 = Bank('local') b2.update() b2.clean_old_sessions() self.assertTrue(len(b2.bank['sessions']) == 1)
def test_session_reload_notover(self): ''' Checks a session is used if present ''' b = Bank('alu') for i in range(1,5): s = Session('alu', self.config, UpdateWorkflow.FLOW) s._session['status'][Workflow.FLOW_INIT] = True b.session = s b.save_session() b = Bank('alu') b.load_session(UpdateWorkflow.FLOW) self.assertTrue(b.session.get_status(Workflow.FLOW_INIT))
def test_clean_old_sessions(self): """ Checks a session is used if present """ b = Bank("local") for i in range(1, 5): s = Session("alu", self.config, UpdateWorkflow.FLOW) s._session["status"][Workflow.FLOW_INIT] = True b.session = s b.save_session() b2 = Bank("local") b2.update() b2.clean_old_sessions() self.assertTrue(len(b2.bank["sessions"]) == 1)
def test_session_reload_notover(self): """ Checks a session is used if present """ b = Bank("alu") for i in range(1, 5): s = Session("alu", self.config, UpdateWorkflow.FLOW) s._session["status"][Workflow.FLOW_INIT] = True b.session = s b.save_session() b = Bank("alu") b.load_session(UpdateWorkflow.FLOW) self.assertTrue(b.session.get_status(Workflow.FLOW_INIT))
def test_session_reload_over(self): """ Checks a session if is not over """ b = Bank('alu') for i in range(1, 5): s = Session('alu', self.config, UpdateWorkflow.FLOW) s._session['status'][Workflow.FLOW_INIT] = True s._session['status'][Workflow.FLOW_OVER] = True b.session = s b.save_session() b = Bank('alu') b.load_session(UpdateWorkflow.FLOW) self.assertFalse(b.session.get_status(Workflow.FLOW_INIT))
def test_session_reload_over(self): """ Checks a session if is not over """ b = Bank('alu') for i in range(1,5): s = Session('alu', self.config, UpdateWorkflow.FLOW) s._session['status'][Workflow.FLOW_INIT] = True s._session['status'][Workflow.FLOW_OVER] = True b.session = s b.save_session() b = Bank('alu') b.load_session(UpdateWorkflow.FLOW) self.assertFalse(b.session.get_status(Workflow.FLOW_INIT))
def migrate_bank(cur, bank, history=False): """ Migrate old MySQL information to the new MongoDB :param cur: MySQL cursor :type cur: MySQL cursor :param bank: Bank name :type bank: String :param history: Keep bank history :type history: Boolean, default False :return: """ query = "SELECT p.path, p.session, p.creation, p.remove, p.size, u.updateRelease, s.logfile from productionDirectory p " query += "JOIN updateBank u on u.idLastSession = p.session JOIN bank b on b.idbank = p.ref_idbank " query += "LEFT JOIN session s ON s.idsession = u.idLastSession " query += "WHERE b.name='" + str(bank) + "' " if not history: query += "AND p.remove IS NULL " query += "ORDER BY p.creation ASC" cur.execute(query) banks = [] not_prod = {} for row in cur.fetchall(): banks.append({ 'path': row[0], 'session': row[1], 'creation': row[2], 'remove': row[3], 'size': row[4], 'release': row[5], 'remoterelease': row[5], 'logfile': row[6] }) # If we want to keep the history we will need to delete from 'production', # session(s) which have been tagged as 'removed', so row[3] is a date if row[3] and history: sess = time.mktime( datetime.datetime.strptime(str(row[2]), "%Y-%m-%d %H:%M:%S").timetuple()) not_prod[sess] = True for prod in banks: pathelts = prod['path'].split('/') release_dir = pathelts[len(pathelts) - 1] prod['prod_dir'] = release_dir pattern = re.compile(".*__(\d+)$") relmatch = pattern.match(release_dir) if relmatch: prod['release'] = prod['release'] + '__' + relmatch.group(1) # We create the session id from the productionDirectory.creation field session_id = time.mktime( datetime.datetime.strptime(str(prod['creation']), "%Y-%m-%d %H:%M:%S").timetuple()) session_exists = False b = Bank(bank, no_log=True) # We check we did not already imported this session into the database for s in b.bank['sessions']: if s['id'] == session_id: logging.warn('Session already imported: ' + b.name + ':' + str(prod['creation'])) session_exists = True break # No need to continue if session_exists: continue for p in b.bank['production']: if p['release'] == prod['release']: logging.warn('Prod release already imported: ' + b.name + ":" + p['release']) continue b.load_session(UpdateWorkflow.FLOW) b.session.set('prod_dir', prod['prod_dir']) b.session.set('action', 'update') b.session.set('release', prod['release']) b.session.set('remoterelease', prod['remoterelease']) # Set production size from productionDirectory.size field b.session.set('fullsize', prod['size']) b.session._session['status'][Workflow.FLOW_OVER] = True b.session._session['update'] = True # We set the session.id (timestamp) with creation field from productionDirectory table b.session.set('id', session_id) b.save_session() # Keep trace of the logfile. We need to do a manual update if prod['logfile'] and os.path.exists(prod['logfile']): b.banks.update({ 'name': b.name, 'sessions.id': session_id }, {'$set': { "sessions.$.log_file": prod['logfile'] }}) # Due to the way save_session set also the production, to exclude last session # from the production entries, we need to loop over each production entries if history: for production in b.bank['production']: if production['session'] in not_prod: b.banks.update( { 'name': b.name, 'production.session': production['session'] }, { '$pull': { 'production': { 'session': production['session'] } } }) # Listing files ? root_files = [] if os.path.exists(prod['path']): root_files = os.listdir(prod['path']) for root_file in root_files: if root_file.startswith('listing.'): fileName, fileExtension = os.path.splitext(root_file) f = open(os.path.join(prod['path'], root_file), 'r') listing = f.read() f.close() f = open( os.path.join(prod['path'], 'listingv1' + fileExtension), 'w') listing = "{\"files\": [], \"name\": \"" + fileExtension.replace( '.', '') + "\"," + listing + "}" f.write(listing) f.close() # Current link? pathelts = prod['path'].split('/') del pathelts[-1] current_link = os.path.join('/'.join(pathelts), 'current') if os.path.lexists(current_link): b.bank['current'] = b.session._session['id'] b.banks.update({'name': b.name}, {'$set': { 'current': b.session._session['id'] }})
def migrate_bank(cur, bank, history=False): """ Migrate old MySQL information to the new MongoDB :param cur: MySQL cursor :type cur: MySQL cursor :param bank: Bank name :type bank: String :param history: Keep bank history :type history: Boolean, default False :return: """ query = "SELECT p.path, p.session, p.creation, p.remove, p.size, u.updateRelease, s.logfile, s.status, r.protocol, " query += "r.server, r.remoteDir FROM productionDirectory p " query += "JOIN updateBank u on u.idLastSession = p.session JOIN bank b on b.idbank = p.ref_idbank " query += "LEFT JOIN session s ON s.idsession = u.idLastSession " query += "JOIN configuration c ON c.idconfiguration = u.ref_idconfiguration JOIN remoteInfo r " query += "ON r.idremoteInfo = c.ref_idremoteInfo " query += "WHERE b.name='" + str(bank) + "' " if not history: query += "AND p.remove IS NULL " query += "ORDER BY p.creation ASC" cur.execute(query) banks = [] not_prod = {} for row in cur.fetchall(): banks.append({ 'path': row[0], 'session': row[1], 'creation': row[2], 'remove': row[3], 'size': humanfriendly.parse_size(row[4].replace(',', '.')), 'release': row[5], 'remoterelease': row[5], 'logfile': row[6], 'status': row[7], 'protocol': row[8], 'server': row[9], 'remotedir': row[10] }) # If we want to keep the history we will need to delete from 'production', # session(s) which have been tagged as 'removed', so row[3] is a date if row[3] and history: sess = time.mktime( datetime.datetime.strptime(str(row[2]), "%Y-%m-%d %H:%M:%S").timetuple()) not_prod[sess] = True for prod in banks: pathelts = prod['path'].split('/') release_dir = pathelts[len(pathelts) - 1] prod['prod_dir'] = release_dir pattern = re.compile(".*__(\d+)$") relmatch = pattern.match(release_dir) if relmatch: prod['release'] = prod['release'] + '__' + relmatch.group(1) # We create the session id from the productionDirectory.creation field session_id = time.mktime( datetime.datetime.strptime(str(prod['creation']), "%Y-%m-%d %H:%M:%S").timetuple()) session_exists = False b = Bank(bank, no_log=True) # We check we did not already imported this session into the database for s in b.bank['sessions']: if s['id'] == session_id: logging.warn('Session already imported: ' + b.name + ':' + str(prod['creation'])) session_exists = True break # No need to continue if session_exists: continue for p in b.bank['production']: if p['release'] == prod['release']: logging.warn('Prod release already imported: ' + b.name + ":" + p['release']) continue b.load_session(UpdateWorkflow.FLOW) b.session.set('prod_dir', prod['prod_dir']) b.session.set('action', 'update') b.session.set('release', prod['release']) b.session.set('remoterelease', prod['remoterelease']) # Biomaj >= 3.0.14 introduce new field in sessions 'workflow_status' # Set production size from productionDirectory.size field b.session.set('workflow_status', True if prod['status'] else False) b.session.set('fullsize', prod['size']) b.session._session['status'][Workflow.FLOW_OVER] = True b.session._session['update'] = True # We set the session.id (timestamp) with creation field from productionDirectory table b.session.set('id', session_id) b.save_session() # We need set update the field 'last_update_time' to the time the bank has been created # because 'save_session' set this value to the time it is called b.banks.update({ 'name': b.name, 'sessions.id': session_id }, {'$set': { 'sessions.$.last_update_time': session_id }}) # Keep trace of the logfile. We need to do a manual update if prod['logfile'] and os.path.exists(prod['logfile']): b.banks.update({ 'name': b.name, 'sessions.id': session_id }, {'$set': { 'sessions.$.log_file': prod['logfile'] }}) # Downloaded and local files (created by 'bank.save_session') # remove as created empty due to 'save_sessions' call cache_dir = b.config.get('cache.dir') download_file = os.path.join(cache_dir, 'files_' + str(session_id)) local_file = os.path.join(cache_dir, 'local_files_' + str(session_id)) if os.path.isfile(download_file): os.unlink(download_file) if os.path.isfile(local_file): os.unlink(local_file) # Due to the way save_session set also the production, to exclude last session # from the production entries, we need to loop over each production entries if history: # If we want to keep history, we also need to keep trace of the time session/update has been deleted # from the database/disk if prod['remove']: removed = time.mktime( datetime.datetime.strptime(str( prod['remove']), "%Y-%m-%d %H:%M:%S").timetuple()) b.banks.update({ 'name': b.name, 'sessions.id': session_id }, {'$set': { 'sessions.$.deleted': removed }}) for production in b.bank['production']: if production['session'] in not_prod: b.banks.update( { 'name': b.name, 'production.session': production['session'] }, { '$pull': { 'production': { 'session': production['session'] } } }) # Listing files ? root_files = [] if os.path.exists(prod['path']): root_files = os.listdir(prod['path']) for root_file in root_files: if root_file.startswith('listing.'): fileName, fileExtension = os.path.splitext(root_file) f = open(os.path.join(prod['path'], root_file), 'r') listing = f.read() f.close() f = open( os.path.join(prod['path'], 'listingv1' + fileExtension), 'w') listing = "{\"files\": [], \"name\": \"" + fileExtension.replace( '.', '') + "\"," + listing + "}" f.write(listing) f.close() if root_file.startswith('flat') and session_id not in not_prod: flat_dir = os.path.join(prod['path'], root_file) if not os.path.exists(flat_dir): print( "[%s] [WARN] Can't list %s/flat directory. Skipping ..." % (b.name, prod['path'])) continue flat_files = os.listdir(flat_dir) files_info = [] for flat_file in flat_files: try: file_info = os.stat(os.path.join(flat_dir, flat_file)) except IOError as err: print( "[%s][WARN] Failed to stat get info from %s: %s" % (b.name, str(file_info), str(err))) continue ctime = datetime.datetime.fromtimestamp(file_info[9]) files_info.append({ 'name': flat_file, 'save_as': flat_file, 'user': file_info[4], 'group': file_info[5], 'year': ctime.year, 'day': ctime.day, 'month': ctime.month, 'root': prod['remotedir'], 'url': prod['protocol'] + '://' + prod['server'], 'size': file_info[6] }) if len(files_info) > 0: # Saved files local_file = open(local_file, 'w') local_file.write(json.dumps(files_info)) local_file.close() # Current link? pathelts = prod['path'].split('/') del pathelts[-1] current_link = os.path.join('/'.join(pathelts), 'current') if os.path.lexists(current_link): b.bank['current'] = b.session._session['id'] b.banks.update({'name': b.name}, {'$set': { 'current': b.session._session['id'] }})