def biomaj_move_production_directories(options): ''' Change bank production directories ''' if not options.bank: return (False, "Bank option is missing") if not os.path.exists(options.newdir): return (False, "Destination directory does not exists") bank = Bank(options.bank, options=options, no_log=True) if not bank.bank['production']: return (False, "Nothing to move, no production directory") bank.load_session(Workflow.FLOW, None) w = Workflow(bank) res = w.wf_init() if not res: return (False, 'Bank initialization failure') for prod in bank.bank['production']: session = bank.get_session_from_release(prod['release']) bank.load_session(Workflow.FLOW, session) prod_path = bank.session.get_full_release_directory() if os.path.exists(prod_path): shutil.move(prod_path, options.newdir) prod['data_dir'] = options.newdir bank.banks.update({'name': options.bank}, {'$set': {'production': bank.bank['production']}}) w.wf_over() return (True, "Bank production directories moved to " + options.newdir + "\nWARNING: do not forget to update accordingly the data.dir and dir.version properties")
def biomaj_status_ko(options, config): ''' Get failed banks ''' banks = Bank.list() banks_list = [] headers = ["Name", "Type(s)", "Release", "Visibility", "Last run"] if not options.json: banks_list.append(headers) for bank in sorted(banks, key=lambda k: k['name']): try: bank = Bank(bank['name'], options=options, no_log=True) bank.load_session(UpdateWorkflow.FLOW) if bank.session is not None: if bank.use_last_session and not bank.session.get_status( Workflow.FLOW_OVER): wf_status = bank.session.get('workflow_status') if wf_status is None or not wf_status: banks_list.append(bank.get_bank_release_info()['info']) except Exception as e: return (False, str(e)) if options.json: msg = {'headers': headers, 'banks': banks_list} return (True, msg) return (True, tabulate(banks_list, headers="firstrow", tablefmt="psql"))
def biomaj_publish(options): ''' Publish a bank ''' if not options.bank: return (False, "Bank name or release is missing") bmaj = Bank(options.bank, options=options, no_log=True) bmaj.load_session() bank = bmaj.bank session = None if options.get_option('release') is None: # Get latest prod release if len(bank['production']) > 0: prod = bank['production'][len(bank['production']) - 1] for s in bank['sessions']: if s['id'] == prod['session']: session = s break else: # Search production release matching release for prod in bank['production']: if prod['release'] == options.release or prod['prod_dir'] == options.release: # Search session related to this production release for s in bank['sessions']: if s['id'] == prod['session']: session = s break break if session is None: return (False, "No production session could be found for this release") bmaj.session._session = session bmaj.publish() return (True, None)
def test_new_session(self): ''' Checks an empty session is created ''' b = Bank('alu') b.load_session(UpdateWorkflow.FLOW) for key in b.session._session['status'].keys(): self.assertFalse(b.session.get_status(key))
def test_computed_nofile(self): b = Bank('computed2') b.load_session(UpdateWorkflow.FLOW) b.session.config.set('protocol', 'none') b.session.config.set('sub1.files.move', 'flat/test_.*') res = b.update(True) self.assertTrue(res) self.assertTrue(os.path.exists(b.session.get_full_release_directory()+'/sub1/flat/test_100.txt'))
def test_extract_release_from_file_content(self): b = Bank("local") b.load_session(UpdateWorkflow.FLOW) b.session.config.set("release.file", "test_100\.txt") b.session.config.set("release.regexp", "Release\s*(\d+)") w = UpdateWorkflow(b) w.wf_release() self.assertTrue(b.session.get("release") == "103")
def test_extract_release_from_file_content(self): b = Bank('local') b.load_session(UpdateWorkflow.FLOW) b.session.config.set('release.file', 'test_100\.txt') b.session.config.set('release.regexp', 'Release\s*(\d+)') w = UpdateWorkflow(b) w.wf_release() self.assertTrue(b.session.get('release') == '103')
def test_new_session(self): """ Checks an empty session is created """ b = Bank("alu") b.load_session(UpdateWorkflow.FLOW) for key in b.session._session["status"].keys(): self.assertFalse(b.session.get_status(key))
def test_remoterelease_check(self): b = Bank('local') b.load_session(ReleaseCheckWorkflow.FLOW) b.session.config.set('release.file', 'test_(\d+)\.txt') b.session.config.set('release.regexp', '') workflow = ReleaseCheckWorkflow(b) res = workflow.start() remoterelease = b.session.get('remoterelease') self.assertTrue(remoterelease == '100')
def test_get_release(self): """ Get release """ b = Bank("alu") b.load_session(UpdateWorkflow.FLOW) res = b.update() self.assertTrue(b.session.get("update")) self.assertTrue(res) self.assertTrue(b.session._session["release"] is not None)
def test_get_release(self): ''' Get release ''' b = Bank('alu') b.load_session(UpdateWorkflow.FLOW) res = b.update() self.assertTrue(b.session.get('update')) self.assertTrue(res) self.assertTrue(b.session._session['release'] is not None)
def test_computed_nofile(self): b = Bank('computed2') b.load_session(UpdateWorkflow.FLOW) b.session.config.set('protocol', 'none') b.session.config.set('sub1.files.move', 'flat/test_.*') res = b.update(True) self.assertTrue(res) self.assertTrue( os.path.exists(b.session.get_full_release_directory() + '/sub1/flat/test_100.txt'))
def biomaj_unpublish(options): ''' Unpublish a bank ''' if not options.bank: return (False, "Bank name is missing") bmaj = Bank(options.bank, options=options, no_log=True) bmaj.load_session() bmaj.unpublish() return (True, None)
def test_session_reload_notover(self): """ Checks a session is used if present """ b = Bank("alu") for i in range(1, 5): s = Session("alu", self.config, UpdateWorkflow.FLOW) s._session["status"][Workflow.FLOW_INIT] = True b.session = s b.save_session() b = Bank("alu") b.load_session(UpdateWorkflow.FLOW) self.assertTrue(b.session.get_status(Workflow.FLOW_INIT))
def test_session_reload_notover(self): ''' Checks a session is used if present ''' b = Bank('alu') for i in range(1,5): s = Session('alu', self.config, UpdateWorkflow.FLOW) s._session['status'][Workflow.FLOW_INIT] = True b.session = s b.save_session() b = Bank('alu') b.load_session(UpdateWorkflow.FLOW) self.assertTrue(b.session.get_status(Workflow.FLOW_INIT))
def test_session_reload_notover(self): ''' Checks a session is used if present ''' b = Bank('alu') for i in range(1, 5): s = Session('alu', self.config, UpdateWorkflow.FLOW) s._session['status'][Workflow.FLOW_INIT] = True b.session = s b.save_session() b = Bank('alu') b.load_session(UpdateWorkflow.FLOW) self.assertTrue(b.session.get_status(Workflow.FLOW_INIT))
def test_session_reload_over(self): """ Checks a session if is not over """ b = Bank('alu') for i in range(1, 5): s = Session('alu', self.config, UpdateWorkflow.FLOW) s._session['status'][Workflow.FLOW_INIT] = True s._session['status'][Workflow.FLOW_OVER] = True b.session = s b.save_session() b = Bank('alu') b.load_session(UpdateWorkflow.FLOW) self.assertFalse(b.session.get_status(Workflow.FLOW_INIT))
def test_session_reload_over(self): """ Checks a session if is not over """ b = Bank('alu') for i in range(1,5): s = Session('alu', self.config, UpdateWorkflow.FLOW) s._session['status'][Workflow.FLOW_INIT] = True s._session['status'][Workflow.FLOW_OVER] = True b.session = s b.save_session() b = Bank('alu') b.load_session(UpdateWorkflow.FLOW) self.assertFalse(b.session.get_status(Workflow.FLOW_INIT))
def test_fromscratch_update_with_release(self): """ Try updating twice, at second time, bank should be updated (force with fromscratch) Use case with release defined in release file """ b = Bank("local") b.load_session(UpdateWorkflow.FLOW) b.session.config.set("release.file", "test_(\d+)\.txt") b.session.config.set("release.regexp", "") w = UpdateWorkflow(b) w.wf_release() self.assertTrue(b.session.get("release") == "100") os.makedirs(b.session.get_full_release_directory()) w = UpdateWorkflow(b) # Reset release b.session.set("release", None) w.options.fromscratch = True w.wf_release() self.assertTrue(b.session.get("release") == "100__1")
def test_fromscratch_update_with_release(self): ''' Try updating twice, at second time, bank should be updated (force with fromscratch) Use case with release defined in release file ''' b = Bank('local') b.load_session(UpdateWorkflow.FLOW) b.session.config.set('release.file', 'test_(\d+)\.txt') b.session.config.set('release.regexp', '') w = UpdateWorkflow(b) w.wf_release() self.assertTrue(b.session.get('release') == '100') os.makedirs(b.session.get_full_release_directory()) w = UpdateWorkflow(b) # Reset release b.session.set('release', None) w.options.fromscratch = True w.wf_release() self.assertTrue(b.session.get('release') == '100__1')
def migrate_bank(cur, bank, history=False): """ Migrate old MySQL information to the new MongoDB :param cur: MySQL cursor :type cur: MySQL cursor :param bank: Bank name :type bank: String :param history: Keep bank history :type history: Boolean, default False :return: """ query = "SELECT p.path, p.session, p.creation, p.remove, p.size, u.updateRelease, s.logfile, s.status, r.protocol, " query += "r.server, r.remoteDir FROM productionDirectory p " query += "JOIN updateBank u on u.idLastSession = p.session JOIN bank b on b.idbank = p.ref_idbank " query += "LEFT JOIN session s ON s.idsession = u.idLastSession " query += "JOIN configuration c ON c.idconfiguration = u.ref_idconfiguration JOIN remoteInfo r " query += "ON r.idremoteInfo = c.ref_idremoteInfo " query += "WHERE b.name='" + str(bank) + "' " if not history: query += "AND p.remove IS NULL " query += "ORDER BY p.creation ASC" cur.execute(query) banks = [] not_prod = {} for row in cur.fetchall(): banks.append({ 'path': row[0], 'session': row[1], 'creation': row[2], 'remove': row[3], 'size': humanfriendly.parse_size(row[4].replace(',', '.')), 'release': row[5], 'remoterelease': row[5], 'logfile': row[6], 'status': row[7], 'protocol': row[8], 'server': row[9], 'remotedir': row[10] }) # If we want to keep the history we will need to delete from 'production', # session(s) which have been tagged as 'removed', so row[3] is a date if row[3] and history: sess = time.mktime( datetime.datetime.strptime(str(row[2]), "%Y-%m-%d %H:%M:%S").timetuple()) not_prod[sess] = True for prod in banks: pathelts = prod['path'].split('/') release_dir = pathelts[len(pathelts) - 1] prod['prod_dir'] = release_dir pattern = re.compile(".*__(\d+)$") relmatch = pattern.match(release_dir) if relmatch: prod['release'] = prod['release'] + '__' + relmatch.group(1) # We create the session id from the productionDirectory.creation field session_id = time.mktime( datetime.datetime.strptime(str(prod['creation']), "%Y-%m-%d %H:%M:%S").timetuple()) session_exists = False b = Bank(bank, no_log=True) # We check we did not already imported this session into the database for s in b.bank['sessions']: if s['id'] == session_id: logging.warn('Session already imported: ' + b.name + ':' + str(prod['creation'])) session_exists = True break # No need to continue if session_exists: continue for p in b.bank['production']: if p['release'] == prod['release']: logging.warn('Prod release already imported: ' + b.name + ":" + p['release']) continue b.load_session(UpdateWorkflow.FLOW) b.session.set('prod_dir', prod['prod_dir']) b.session.set('action', 'update') b.session.set('release', prod['release']) b.session.set('remoterelease', prod['remoterelease']) # Biomaj >= 3.0.14 introduce new field in sessions 'workflow_status' # Set production size from productionDirectory.size field b.session.set('workflow_status', True if prod['status'] else False) b.session.set('fullsize', prod['size']) b.session._session['status'][Workflow.FLOW_OVER] = True b.session._session['update'] = True # We set the session.id (timestamp) with creation field from productionDirectory table b.session.set('id', session_id) b.save_session() # We need set update the field 'last_update_time' to the time the bank has been created # because 'save_session' set this value to the time it is called b.banks.update({ 'name': b.name, 'sessions.id': session_id }, {'$set': { 'sessions.$.last_update_time': session_id }}) # Keep trace of the logfile. We need to do a manual update if prod['logfile'] and os.path.exists(prod['logfile']): b.banks.update({ 'name': b.name, 'sessions.id': session_id }, {'$set': { 'sessions.$.log_file': prod['logfile'] }}) # Downloaded and local files (created by 'bank.save_session') # remove as created empty due to 'save_sessions' call cache_dir = b.config.get('cache.dir') download_file = os.path.join(cache_dir, 'files_' + str(session_id)) local_file = os.path.join(cache_dir, 'local_files_' + str(session_id)) if os.path.isfile(download_file): os.unlink(download_file) if os.path.isfile(local_file): os.unlink(local_file) # Due to the way save_session set also the production, to exclude last session # from the production entries, we need to loop over each production entries if history: # If we want to keep history, we also need to keep trace of the time session/update has been deleted # from the database/disk if prod['remove']: removed = time.mktime( datetime.datetime.strptime(str( prod['remove']), "%Y-%m-%d %H:%M:%S").timetuple()) b.banks.update({ 'name': b.name, 'sessions.id': session_id }, {'$set': { 'sessions.$.deleted': removed }}) for production in b.bank['production']: if production['session'] in not_prod: b.banks.update( { 'name': b.name, 'production.session': production['session'] }, { '$pull': { 'production': { 'session': production['session'] } } }) # Listing files ? root_files = [] if os.path.exists(prod['path']): root_files = os.listdir(prod['path']) for root_file in root_files: if root_file.startswith('listing.'): fileName, fileExtension = os.path.splitext(root_file) f = open(os.path.join(prod['path'], root_file), 'r') listing = f.read() f.close() f = open( os.path.join(prod['path'], 'listingv1' + fileExtension), 'w') listing = "{\"files\": [], \"name\": \"" + fileExtension.replace( '.', '') + "\"," + listing + "}" f.write(listing) f.close() if root_file.startswith('flat') and session_id not in not_prod: flat_dir = os.path.join(prod['path'], root_file) if not os.path.exists(flat_dir): print( "[%s] [WARN] Can't list %s/flat directory. Skipping ..." % (b.name, prod['path'])) continue flat_files = os.listdir(flat_dir) files_info = [] for flat_file in flat_files: try: file_info = os.stat(os.path.join(flat_dir, flat_file)) except IOError as err: print( "[%s][WARN] Failed to stat get info from %s: %s" % (b.name, str(file_info), str(err))) continue ctime = datetime.datetime.fromtimestamp(file_info[9]) files_info.append({ 'name': flat_file, 'save_as': flat_file, 'user': file_info[4], 'group': file_info[5], 'year': ctime.year, 'day': ctime.day, 'month': ctime.month, 'root': prod['remotedir'], 'url': prod['protocol'] + '://' + prod['server'], 'size': file_info[6] }) if len(files_info) > 0: # Saved files local_file = open(local_file, 'w') local_file.write(json.dumps(files_info)) local_file.close() # Current link? pathelts = prod['path'].split('/') del pathelts[-1] current_link = os.path.join('/'.join(pathelts), 'current') if os.path.lexists(current_link): b.bank['current'] = b.session._session['id'] b.banks.update({'name': b.name}, {'$set': { 'current': b.session._session['id'] }})
def main(): parser = argparse.ArgumentParser(add_help=False) parser.add_argument('-c', '--config', dest="config", help="Configuration file") parser.add_argument('--check', dest="check", help="Check bank property file", action="store_true", default=False) parser.add_argument('-u', '--update', dest="update", help="Update action", action="store_true", default=False) parser.add_argument('--fromscratch', dest="fromscratch", help="Force a new cycle update", action="store_true", default=False) parser.add_argument('-z', '--from-scratch', dest="fromscratch", help="Force a new cycle update", action="store_true", default=False) parser.add_argument('-p', '--publish', dest="publish", help="Publish", action="store_true", default=False) parser.add_argument('--unpublish', dest="unpublish", help="Unpublish", action="store_true", default=False) parser.add_argument('--release', dest="release", help="release of the bank") parser.add_argument( '--from-task', dest="from_task", help="Start cycle at a specific task (init always executed)") parser.add_argument( '--process', dest="process", help= "Linked to from-task, optionally specify a block, meta or process name to start from" ) parser.add_argument('-l', '--log', dest="log", help="log level") parser.add_argument('-r', '--remove', dest="remove", help="Remove a bank release", action="store_true", default=False) parser.add_argument('--remove-all', dest="removeall", help="Remove all bank releases and database records", action="store_true", default=False) parser.add_argument('--remove-pending', dest="removepending", help="Remove pending release", action="store_true", default=False) parser.add_argument('-s', '--status', dest="status", help="Get status", action="store_true", default=False) parser.add_argument('-b', '--bank', dest="bank", help="bank name") parser.add_argument('--owner', dest="owner", help="change owner of the bank") parser.add_argument('--stop-before', dest="stop_before", help="Store workflow before task") parser.add_argument('--stop-after', dest="stop_after", help="Store workflow after task") parser.add_argument('--freeze', dest="freeze", help="Freeze a bank release", action="store_true", default=False) parser.add_argument('--unfreeze', dest="unfreeze", help="Unfreeze a bank release", action="store_true", default=False) parser.add_argument('-f', '--force', dest="force", help="Force action", action="store_true", default=False) parser.add_argument('-h', '--help', dest="help", help="Show usage", action="store_true", default=False) parser.add_argument('--search', dest="search", help="Search by format and types", action="store_true", default=False) parser.add_argument('--formats', dest="formats", help="List of formats to search, comma separated") parser.add_argument('--types', dest="types", help="List of types to search, comma separated") parser.add_argument('--query', dest="query", help="Lucene query syntax to search in index") parser.add_argument('--show', dest="show", help="Show format files for selected bank", action="store_true", default=False) parser.add_argument('-n', '--change-dbname', dest="newbank", help="Change old bank name to this new bank name") parser.add_argument( '-e', '--move-production-directories', dest="newdir", help= "Change bank production directories location to this new path, path must exists" ) parser.add_argument('--visibility', dest="visibility", help="visibility status of the bank") parser.add_argument('--maintenance', dest="maintenance", help="Maintenance mode (on/off/status)") parser.add_argument('--version', dest="version", help="Show version", action="store_true", default=False) parser.add_argument('--status-ko', dest="statusko", help="Get bank in KO status", action="store_true", default=False) options = Options() parser.parse_args(namespace=options) options.no_log = False if options.help: print(''' --config: global.properties file path --status: list of banks with published release [OPTIONAL] --bank xx / bank: Get status details of bank --status-ko: list of banks in error status (last run) --log DEBUG|INFO|WARN|ERR [OPTIONAL]: set log level in logs for this run, default is set in global.properties file --check: Check bank property file [MANDATORY] --bank xx: name of the bank to check (will check xx.properties) --owner yy: Change owner of the bank (user id) [MANDATORY] --bank xx: name of the bank --visibility public|private: change visibility public/private of a bank [MANDATORY] --bank xx: name of the bank --change-dbname yy: Change name of the bank to this new name [MANDATORY] --bank xx: current name of the bank --move-production-directories yy: Change bank production directories location to this new path, path must exists [MANDATORY] --bank xx: current name of the bank --update: Update bank [MANDATORY] --bank xx: name of the bank(s) to update, comma separated [OPTIONAL] --publish: after update set as *current* version --from-scratch: force a new update cycle, even if release is identical, release will be incremented like (myrel_1) --stop-before xx: stop update cycle before the start of step xx --stop-after xx: stop update cycle after step xx has completed --from-task xx --release yy: Force an re-update cycle for bank release *yy* or from current cycle (in production directories), skipping steps up to *xx* --process xx: linked to from-task, optionally specify a block, meta or process name to start from --release xx: release to update --publish: Publish bank as current release to use [MANDATORY] --bank xx: name of the bank to update --release xx: release of the bank to publish --unpublish: Unpublish bank (remove current) [MANDATORY] --bank xx: name of the bank to update --remove-all: Remove all bank releases and database records [MANDATORY] --bank xx: name of the bank to update [OPTIONAL] --force: remove freezed releases --remove-pending: Remove pending releases [MANDATORY] --bank xx: name of the bank to update --remove: Remove bank release (files and database release) [MANDATORY] --bank xx: name of the bank to update --release xx: release of the bank to remove Release must not be the *current* version. If this is the case, publish a new release before. --freeze: Freeze bank release (cannot be removed) [MANDATORY] --bank xx: name of the bank to update --release xx: release of the bank to remove --unfreeze: Unfreeze bank release (can be removed) [MANDATORY] --bank xx: name of the bank to update --release xx: release of the bank to remove --search: basic search in bank production releases, return list of banks --formats xx,yy : list of comma separated format AND/OR --types xx,yy : list of comma separated type --query "LUCENE query syntax": search in index (if activated) --show: Show bank files per format [MANDATORY] --bank xx: name of the bank to show [OPTIONAL] --release xx: release of the bank to show --maintenance on/off/status: (un)set biomaj in maintenance mode to prevent updates/removal ''') return if options.version: version = pkg_resources.require('biomaj')[0].version print('Version: ' + str(version)) return if options.stop_after or options.stop_before or options.from_task: available_steps = [] for flow in UpdateWorkflow.FLOW: available_steps.append(flow['name']) for flow in RemoveWorkflow.FLOW: available_steps.append(flow['name']) if options.stop_after: if options.stop_after not in available_steps: print('Invalid step: ' + options.stop_after) sys.exit(1) if options.stop_before: if options.stop_before not in available_steps: print('Invalid step: ' + options.stop_before) sys.exit(1) if options.from_task: if options.from_task not in available_steps: print('Invalid step: ' + options.from_task) sys.exit(1) bmaj = None try: if options.config is not None: BiomajConfig.load_config(options.config) else: BiomajConfig.load_config() except Exception as e: print(str(e)) sys.exit(1) try: if options.maintenance: if options.maintenance not in ['on', 'off', 'status']: print("Wrong maintenance value [on,off,status]") sys.exit(1) data_dir = BiomajConfig.global_config.get('GENERAL', 'data.dir') if BiomajConfig.global_config.has_option('GENERAL', 'lock.dir'): lock_dir = BiomajConfig.global_config.get( 'GENERAL', 'lock.dir') else: lock_dir = data_dir maintenance_lock_file = os.path.join(lock_dir, 'biomaj.lock') if options.maintenance == 'status': if os.path.exists(maintenance_lock_file): print("Maintenance: On") else: print("Maintenance: Off") sys.exit(0) if options.maintenance == 'on': f = open(maintenance_lock_file, 'w') f.write('1') f.close() print("Maintenance set to On") sys.exit(0) if options.maintenance == 'off': if os.path.exists(maintenance_lock_file): os.remove(maintenance_lock_file) print("Maintenance set to Off") sys.exit(0) if options.owner: if not options.bank: print("Bank option is missing") sys.exit(1) bank = Bank(options.bank, no_log=True) bank.set_owner(options.owner) sys.exit(0) if options.visibility: if not options.bank: print("Bank option is missing") sys.exit(1) if options.visibility not in ['public', 'private']: print("Valid values are public|private") sys.exit(1) bank = Bank(options.bank, no_log=True) bank.set_visibility(options.visibility) print( "Do not forget to update accordingly the visibility.default parameter in the configuration file" ) sys.exit(0) if options.newdir: if not options.bank: print("Bank option is missing") sys.exit(1) if not os.path.exists(options.newdir): print("Destination directory does not exists") bank = Bank(options.bank, options=options, no_log=True) if not bank.bank['production']: print("Nothing to move, no production directory") sys.exit(0) bank.load_session(Workflow.FLOW, None) w = Workflow(bank) res = w.wf_init() if not res: sys.exit(1) for prod in bank.bank['production']: session = bank.get_session_from_release(prod['release']) bank.load_session(Workflow.FLOW, session) prod_path = bank.session.get_full_release_directory() if os.path.exists(prod_path): shutil.move(prod_path, options.newdir) prod['data_dir'] = options.newdir bank.banks.update( {'name': options.bank}, {'$set': { 'production': bank.bank['production'] }}) print("Bank production directories moved to " + options.newdir) print( "WARNING: do not forget to update accordingly the data.dir and dir.version properties" ) w.wf_over() sys.exit(0) if options.newbank: if not options.bank: print("Bank option is missing") sys.exit(1) bank = Bank(options.bank, no_log=True) conf_dir = BiomajConfig.global_config.get('GENERAL', 'conf.dir') bank_prop_file = os.path.join(conf_dir, options.bank + '.properties') config_bank = configparser.SafeConfigParser() config_bank.read( [os.path.join(conf_dir, options.bank + '.properties')]) config_bank.set('GENERAL', 'db.name', options.newbank) newbank_prop_file = open( os.path.join(conf_dir, options.newbank + '.properties'), 'w') config_bank.write(newbank_prop_file) newbank_prop_file.close() bank.banks.update({'name': options.bank}, {'$set': { 'name': options.newbank }}) os.remove(bank_prop_file) print("Bank " + options.bank + " renamed to " + options.newbank) sys.exit(0) if options.search: if options.query: res = Bank.searchindex(options.query) print("Query matches for :" + options.query) results = [["Release", "Format(s)", "Type(s)", "Files"]] for match in res: results.append([ match['_source']['release'], str(match['_source']['format']), str(match['_source']['types']), ','.join(match['_source']['files']) ]) print(tabulate(results, headers="firstrow", tablefmt="grid")) else: formats = [] if options.formats: formats = options.formats.split(',') types = [] if options.types: types = options.types.split(',') print("Search by formats=" + str(formats) + ", types=" + str(types)) res = Bank.search(formats, types, False) results = [[ "Name", "Release", "Format(s)", "Type(s)", 'Published' ]] for bank in sorted(res, key=lambda bank: (bank['name'])): b = bank['name'] bank['production'].sort(key=lambda n: n['release'], reverse=True) for prod in bank['production']: iscurrent = "" if prod['session'] == bank['current']: iscurrent = "yes" results.append([ b if b else '', prod['release'], ','.join(prod['formats']), ','.join(prod['types']), iscurrent ]) print(tabulate(results, headers="firstrow", tablefmt="grid")) sys.exit(0) if options.show: if not options.bank: print("Bank option is required") sys.exit(1) bank = Bank(options.bank, no_log=True) results = [[ "Name", "Release", "Format(s)", "Type(s)", "Tag(s)", "File(s)" ]] current = None fformat = None if 'current' in bank.bank and bank.bank['current']: current = bank.bank['current'] for prod in bank.bank['production']: include = True release = prod['release'] if current == prod['session']: release += ' (current)' if options.release and (prod['release'] != options.release and prod['prod_dir'] != options.release): include = False if include: session = bank.get_session_from_release(prod['release']) formats = session['formats'] afiles = [] atags = [] atypes = [] for fformat in list(formats.keys()): for elt in formats[fformat]: atypes.append(','.join(elt['types'])) for tag in list(elt['tags'].keys()): atags.append(elt['tags'][tag]) for eltfile in elt['files']: afiles.append(eltfile) results.append([ bank.bank['name'], release, fformat, ','.join(atypes), ','.join(atags), ','.join(afiles) ]) print(tabulate(results, headers="firstrow", tablefmt="grid")) sys.exit(0) if options.check: if not options.bank: print("Bank name is missing") sys.exit(1) bank = Bank(options.bank, no_log=True) print(options.bank + " check: " + str(bank.check()) + "\n") sys.exit(0) if options.status: if options.bank: bank = Bank(options.bank, no_log=True) info = bank.get_bank_release_info(full=True) print( tabulate(info['info'], headers='firstrow', tablefmt='psql')) print( tabulate(info['prod'], headers='firstrow', tablefmt='psql')) # do we have some pending release(s) if 'pend' in info and len(info['pend']) > 1: print( tabulate(info['pend'], headers='firstrow', tablefmt='psql')) else: banks = Bank.list() # Headers of output table banks_list = [["Name", "Type(s)", "Release", "Visibility"]] for bank in sorted(banks, key=lambda k: k['name']): bank = Bank(bank['name'], no_log=True) banks_list.append(bank.get_bank_release_info()['info']) print(tabulate(banks_list, headers="firstrow", tablefmt="psql")) sys.exit(0) if options.statusko: banks = Bank.list() banks_list = [[ "Name", "Type(s)", "Release", "Visibility", "Last run" ]] for bank in sorted(banks, key=lambda k: k['name']): try: bank = Bank(bank['name'], no_log=True) bank.load_session(UpdateWorkflow.FLOW) if bank.session is not None: if bank.use_last_session and not bank.session.get_status( Workflow.FLOW_OVER): wf_status = bank.session.get('workflow_status') if wf_status is None or not wf_status: banks_list.append( bank.get_bank_release_info()['info']) except Exception as e: print(str(e)) print(tabulate(banks_list, headers="firstrow", tablefmt="psql")) if options.update: if not options.bank: print("Bank name is missing") sys.exit(1) banks = options.bank.split(',') gres = True for bank in banks: options.bank = bank bmaj = Bank(bank, options) print('Log file: ' + bmaj.config.log_file) check_status = bmaj.check() if not check_status: print('Skip bank ' + options.bank + ': wrong config') gres = False continue res = bmaj.update(depends=True) if not res: gres = False Notify.notifyBankAction(bmaj) if not gres: sys.exit(1) if options.freeze: if not options.bank: print("Bank name is missing") sys.exit(1) if not options.release: print("Bank release is missing") sys.exit(1) bmaj = Bank(options.bank, options) res = bmaj.freeze(options.release) if not res: sys.exit(1) if options.unfreeze: if not options.bank: print("Bank name is missing") sys.exit(1) if not options.release: print("Bank release is missing") sys.exit(1) bmaj = Bank(options.bank, options) res = bmaj.unfreeze(options.release) if not res: sys.exit(1) if options.remove or options.removeall: if not options.bank: print("Bank name is missing") sys.exit(1) if options.remove and not options.release: print("Bank release is missing") sys.exit(1) if options.removeall: bmaj = Bank(options.bank, options, no_log=True) print('Log file: ' + bmaj.config.log_file) res = bmaj.removeAll(options.force) else: bmaj = Bank(options.bank, options) print('Log file: ' + bmaj.config.log_file) res = bmaj.remove(options.release) Notify.notifyBankAction(bmaj) if not res: sys.exit(1) if options.removepending: if not options.bank: print("Bank name is missing") sys.exit(1) bmaj = Bank(options.bank, options, no_log=True) res = bmaj.remove_pending(options.release) if not res: sys.exit(1) if options.unpublish: if not options.bank: print("Bank name is missing") sys.exit(1) bmaj = Bank(options.bank, options, no_log=True) bmaj.load_session() bmaj.unpublish() sys.exit(0) if options.publish: if not options.bank: print("Bank name or release is missing") sys.exit(1) bmaj = Bank(options.bank, options, no_log=True) bmaj.load_session() bank = bmaj.bank session = None if options.get_option('release') is None: # Get latest prod release if len(bank['production']) > 0: prod = bank['production'][len(bank['production']) - 1] for s in bank['sessions']: if s['id'] == prod['session']: session = s break else: # Search production release matching release for prod in bank['production']: if prod['release'] == options.release or prod[ 'prod_dir'] == options.release: # Search session related to this production release for s in bank['sessions']: if s['id'] == prod['session']: session = s break break if session is None: print("No production session could be found for this release") sys.exit(1) bmaj.session._session = session bmaj.publish() except Exception as e: print(str(e))
def migrate_bank(cur, bank, history=False): """ Migrate old MySQL information to the new MongoDB :param cur: MySQL cursor :type cur: MySQL cursor :param bank: Bank name :type bank: String :param history: Keep bank history :type history: Boolean, default False :return: """ query = "SELECT p.path, p.session, p.creation, p.remove, p.size, u.updateRelease, s.logfile from productionDirectory p " query += "JOIN updateBank u on u.idLastSession = p.session JOIN bank b on b.idbank = p.ref_idbank " query += "LEFT JOIN session s ON s.idsession = u.idLastSession " query += "WHERE b.name='" + str(bank) + "' " if not history: query += "AND p.remove IS NULL " query += "ORDER BY p.creation ASC" cur.execute(query) banks = [] not_prod = {} for row in cur.fetchall(): banks.append({ 'path': row[0], 'session': row[1], 'creation': row[2], 'remove': row[3], 'size': row[4], 'release': row[5], 'remoterelease': row[5], 'logfile': row[6] }) # If we want to keep the history we will need to delete from 'production', # session(s) which have been tagged as 'removed', so row[3] is a date if row[3] and history: sess = time.mktime( datetime.datetime.strptime(str(row[2]), "%Y-%m-%d %H:%M:%S").timetuple()) not_prod[sess] = True for prod in banks: pathelts = prod['path'].split('/') release_dir = pathelts[len(pathelts) - 1] prod['prod_dir'] = release_dir pattern = re.compile(".*__(\d+)$") relmatch = pattern.match(release_dir) if relmatch: prod['release'] = prod['release'] + '__' + relmatch.group(1) # We create the session id from the productionDirectory.creation field session_id = time.mktime( datetime.datetime.strptime(str(prod['creation']), "%Y-%m-%d %H:%M:%S").timetuple()) session_exists = False b = Bank(bank, no_log=True) # We check we did not already imported this session into the database for s in b.bank['sessions']: if s['id'] == session_id: logging.warn('Session already imported: ' + b.name + ':' + str(prod['creation'])) session_exists = True break # No need to continue if session_exists: continue for p in b.bank['production']: if p['release'] == prod['release']: logging.warn('Prod release already imported: ' + b.name + ":" + p['release']) continue b.load_session(UpdateWorkflow.FLOW) b.session.set('prod_dir', prod['prod_dir']) b.session.set('action', 'update') b.session.set('release', prod['release']) b.session.set('remoterelease', prod['remoterelease']) # Set production size from productionDirectory.size field b.session.set('fullsize', prod['size']) b.session._session['status'][Workflow.FLOW_OVER] = True b.session._session['update'] = True # We set the session.id (timestamp) with creation field from productionDirectory table b.session.set('id', session_id) b.save_session() # Keep trace of the logfile. We need to do a manual update if prod['logfile'] and os.path.exists(prod['logfile']): b.banks.update({ 'name': b.name, 'sessions.id': session_id }, {'$set': { "sessions.$.log_file": prod['logfile'] }}) # Due to the way save_session set also the production, to exclude last session # from the production entries, we need to loop over each production entries if history: for production in b.bank['production']: if production['session'] in not_prod: b.banks.update( { 'name': b.name, 'production.session': production['session'] }, { '$pull': { 'production': { 'session': production['session'] } } }) # Listing files ? root_files = [] if os.path.exists(prod['path']): root_files = os.listdir(prod['path']) for root_file in root_files: if root_file.startswith('listing.'): fileName, fileExtension = os.path.splitext(root_file) f = open(os.path.join(prod['path'], root_file), 'r') listing = f.read() f.close() f = open( os.path.join(prod['path'], 'listingv1' + fileExtension), 'w') listing = "{\"files\": [], \"name\": \"" + fileExtension.replace( '.', '') + "\"," + listing + "}" f.write(listing) f.close() # Current link? pathelts = prod['path'].split('/') del pathelts[-1] current_link = os.path.join('/'.join(pathelts), 'current') if os.path.lexists(current_link): b.bank['current'] = b.session._session['id'] b.banks.update({'name': b.name}, {'$set': { 'current': b.session._session['id'] }})
def main(): parser = argparse.ArgumentParser(add_help=False) parser.add_argument('-c', '--config', dest="config",help="Configuration file") parser.add_argument('--check', dest="check", help="Check bank property file", action="store_true", default=False) parser.add_argument('-u', '--update', dest="update", help="Update action", action="store_true", default=False) parser.add_argument('--fromscratch', dest="fromscratch", help="Force a new cycle update", action="store_true", default=False) parser.add_argument('-z', '--from-scratch', dest="fromscratch", help="Force a new cycle update", action="store_true", default=False) parser.add_argument('-p', '--publish', dest="publish", help="Publish", action="store_true", default=False) parser.add_argument('--unpublish', dest="unpublish", help="Unpublish", action="store_true", default=False) parser.add_argument('--release', dest="release", help="release of the bank") parser.add_argument('--from-task', dest="from_task", help="Start cycle at a specific task (init always executed)") parser.add_argument('--process', dest="process", help="Linked to from-task, optionally specify a block, meta or process name to start from") parser.add_argument('-l', '--log', dest="log", help="log level") parser.add_argument('-r', '--remove', dest="remove", help="Remove a bank release", action="store_true", default=False) parser.add_argument('--remove-all', dest="removeall", help="Remove all bank releases and database records", action="store_true", default=False) parser.add_argument('--remove-pending', dest="removepending", help="Remove pending release", action="store_true", default=False) parser.add_argument('-s', '--status', dest="status", help="Get status", action="store_true", default=False) parser.add_argument('-b', '--bank', dest="bank", help="bank name") parser.add_argument('--owner', dest="owner", help="change owner of the bank") parser.add_argument('--stop-before', dest="stop_before", help="Store workflow before task") parser.add_argument('--stop-after', dest="stop_after", help="Store workflow after task") parser.add_argument('--freeze', dest="freeze", help="Freeze a bank release", action="store_true", default=False) parser.add_argument('--unfreeze', dest="unfreeze", help="Unfreeze a bank release", action="store_true", default=False) parser.add_argument('-f', '--force', dest="force", help="Force action", action="store_true", default=False) parser.add_argument('-h', '--help', dest="help", help="Show usage", action="store_true", default=False) parser.add_argument('--search', dest="search", help="Search by format and types", action="store_true", default=False) parser.add_argument('--formats', dest="formats", help="List of formats to search, comma separated") parser.add_argument('--types', dest="types", help="List of types to search, comma separated") parser.add_argument('--query', dest="query", help="Lucene query syntax to search in index") parser.add_argument('--show', dest="show", help="Show format files for selected bank", action="store_true", default=False) parser.add_argument('-n', '--change-dbname', dest="newbank", help="Change old bank name to this new bank name") parser.add_argument('-e', '--move-production-directories', dest="newdir",help="Change bank production directories location to this new path, path must exists") parser.add_argument('--visibility', dest="visibility", help="visibility status of the bank") parser.add_argument('--maintenance', dest="maintenance", help="Maintenance mode (on/off/status)") parser.add_argument('--version', dest="version", help="Show version", action="store_true", default=False) parser.add_argument('--status-ko', dest="statusko", help="Get bank in KO status", action="store_true", default=False) options = Options() parser.parse_args(namespace=options) options.no_log = False if options.help: print(''' --config: global.properties file path --status: list of banks with published release [OPTIONAL] --bank xx / bank: Get status details of bank --status-ko: list of banks in error status (last run) --log DEBUG|INFO|WARN|ERR [OPTIONAL]: set log level in logs for this run, default is set in global.properties file --check: Check bank property file [MANDATORY] --bank xx: name of the bank to check (will check xx.properties) --owner yy: Change owner of the bank (user id) [MANDATORY] --bank xx: name of the bank --visibility public|private: change visibility public/private of a bank [MANDATORY] --bank xx: name of the bank --change-dbname yy: Change name of the bank to this new name [MANDATORY] --bank xx: current name of the bank --move-production-directories yy: Change bank production directories location to this new path, path must exists [MANDATORY] --bank xx: current name of the bank --update: Update bank [MANDATORY] --bank xx: name of the bank(s) to update, comma separated [OPTIONAL] --publish: after update set as *current* version --from-scratch: force a new update cycle, even if release is identical, release will be incremented like (myrel_1) --stop-before xx: stop update cycle before the start of step xx --stop-after xx: stop update cycle after step xx has completed --from-task xx --release yy: Force an re-update cycle for bank release *yy* or from current cycle (in production directories), skipping steps up to *xx* --process xx: linked to from-task, optionally specify a block, meta or process name to start from --release xx: release to update --publish: Publish bank as current release to use [MANDATORY] --bank xx: name of the bank to update --release xx: release of the bank to publish --unpublish: Unpublish bank (remove current) [MANDATORY] --bank xx: name of the bank to update --remove-all: Remove all bank releases and database records [MANDATORY] --bank xx: name of the bank to update [OPTIONAL] --force: remove freezed releases --remove-pending: Remove pending releases [MANDATORY] --bank xx: name of the bank to update --remove: Remove bank release (files and database release) [MANDATORY] --bank xx: name of the bank to update --release xx: release of the bank to remove Release must not be the *current* version. If this is the case, publish a new release before. --freeze: Freeze bank release (cannot be removed) [MANDATORY] --bank xx: name of the bank to update --release xx: release of the bank to remove --unfreeze: Unfreeze bank release (can be removed) [MANDATORY] --bank xx: name of the bank to update --release xx: release of the bank to remove --search: basic search in bank production releases, return list of banks --formats xx,yy : list of comma separated format AND/OR --types xx,yy : list of comma separated type --query "LUCENE query syntax": search in index (if activated) --show: Show bank files per format [MANDATORY] --bank xx: name of the bank to show [OPTIONAL] --release xx: release of the bank to show --maintenance on/off/status: (un)set biomaj in maintenance mode to prevent updates/removal ''') return if options.version: version = pkg_resources.require('biomaj')[0].version print('Version: '+str(version)) return if options.stop_after or options.stop_before or options.from_task: available_steps = [] for flow in UpdateWorkflow.FLOW: available_steps.append(flow['name']) for flow in RemoveWorkflow.FLOW: available_steps.append(flow['name']) if options.stop_after: if options.stop_after not in available_steps: print('Invalid step: '+options.stop_after) sys.exit(1) if options.stop_before: if options.stop_before not in available_steps: print('Invalid step: '+options.stop_before) sys.exit(1) if options.from_task: if options.from_task not in available_steps: print('Invalid step: '+options.from_task) sys.exit(1) bmaj = None try: if options.config is not None: BiomajConfig.load_config(options.config) else: BiomajConfig.load_config() except Exception as e: print(str(e)) sys.exit(1) try: if options.maintenance: if options.maintenance not in ['on', 'off', 'status']: print("Wrong maintenance value [on,off,status]") sys.exit(1) data_dir = BiomajConfig.global_config.get('GENERAL', 'data.dir') if BiomajConfig.global_config.has_option('GENERAL', 'lock.dir'): lock_dir = BiomajConfig.global_config.get('GENERAL', 'lock.dir') else: lock_dir = data_dir maintenance_lock_file = os.path.join(lock_dir,'biomaj.lock') if options.maintenance == 'status': if os.path.exists(maintenance_lock_file): print("Maintenance: On") else: print("Maintenance: Off") sys.exit(0) if options.maintenance == 'on': f = open(maintenance_lock_file, 'w') f.write('1') f.close() print("Maintenance set to On") sys.exit(0) if options.maintenance == 'off': if os.path.exists(maintenance_lock_file): os.remove(maintenance_lock_file) print("Maintenance set to Off") sys.exit(0) if options.owner: if not options.bank: print("Bank option is missing") sys.exit(1) bank = Bank(options.bank, no_log=True) bank.set_owner(options.owner) sys.exit(0) if options.visibility: if not options.bank: print("Bank option is missing") sys.exit(1) if options.visibility not in ['public', 'private']: print("Valid values are public|private") sys.exit(1) bank = Bank(options.bank, no_log=True) bank.set_visibility(options.visibility) print("Do not forget to update accordingly the visibility.default parameter in the configuration file") sys.exit(0) if options.newdir: if not options.bank: print("Bank option is missing") sys.exit(1) if not os.path.exists(options.newdir): print("Destination directory does not exists") bank = Bank(options.bank, options=options, no_log=True) if not bank.bank['production']: print("Nothing to move, no production directory") sys.exit(0) bank.load_session(Workflow.FLOW, None) w = Workflow(bank) res = w.wf_init() if not res: sys.exit(1) for prod in bank.bank['production']: session = bank.get_session_from_release(prod['release']) bank.load_session(Workflow.FLOW, session) prod_path = bank.session.get_full_release_directory() if os.path.exists(prod_path): shutil.move(prod_path, options.newdir) prod['data_dir'] = options.newdir bank.banks.update({'name': options.bank}, {'$set' : { 'production': bank.bank['production'] }}) print("Bank production directories moved to " + options.newdir) print("WARNING: do not forget to update accordingly the data.dir and dir.version properties") w.wf_over() sys.exit(0) if options.newbank: if not options.bank: print("Bank option is missing") sys.exit(1) bank = Bank(options.bank, no_log=True) conf_dir = BiomajConfig.global_config.get('GENERAL', 'conf.dir') bank_prop_file = os.path.join(conf_dir,options.bank+'.properties') config_bank = configparser.SafeConfigParser() config_bank.read([os.path.join(conf_dir,options.bank+'.properties')]) config_bank.set('GENERAL', 'db.name', options.newbank) newbank_prop_file = open(os.path.join(conf_dir,options.newbank+'.properties'),'w') config_bank.write(newbank_prop_file) newbank_prop_file.close() bank.banks.update({'name': options.bank}, {'$set' : { 'name': options.newbank }}) os.remove(bank_prop_file) print("Bank "+options.bank+" renamed to "+options.newbank) sys.exit(0) if options.search: if options.query: res = Bank.searchindex(options.query) print("Query matches for :"+options.query) results = [["Release", "Format(s)", "Type(s)", "Files"]] for match in res: results.append([match['_source']['release'], str(match['_source']['format']), str(match['_source']['types']), ','.join(match['_source']['files'])]) print(tabulate(results, headers="firstrow", tablefmt="grid")) else: formats = [] if options.formats: formats = options.formats.split(',') types = [] if options.types: types = options.types.split(',') print("Search by formats="+str(formats)+", types="+str(types)) res = Bank.search(formats, types, False) results = [["Name", "Release", "Format(s)", "Type(s)", 'Current']] for bank in sorted(res, key=lambda bank: (bank['name'])): b = bank['name'] bank['production'].sort(key=lambda n: n['release'], reverse=True) for prod in bank['production']: iscurrent = "" if prod['session'] == bank['current']: iscurrent = "yes" results.append([b if b else '', prod['release'], ','.join(prod['formats']), ','.join(prod['types']), iscurrent]) b = None print(tabulate(results, headers="firstrow", tablefmt="grid")) sys.exit(0) if options.show: if not options.bank: print("Bank option is required") sys.exit(1) bank = Bank(options.bank, no_log=True) results = [["Name", "Release", "Format(s)", "Type(s)", "Tag(s)", "File(s)"]] current = None fformat = None if 'current' in bank.bank and bank.bank['current']: current = bank.bank['current'] for prod in bank.bank['production']: include = True release = prod['release'] if current == prod['session']: release += ' (current)' if options.release and (prod['release'] != options.release and prod['prod_dir'] != options.release): include =False if include: session = bank.get_session_from_release(prod['release']) formats = session['formats'] afiles = [] atags = [] atypes = [] for fformat in list(formats.keys()): for elt in formats[fformat]: atypes.append(','.join(elt['types'])) for tag in list(elt['tags'].keys()): atags.append(elt['tags'][tag]) for eltfile in elt['files']: afiles.append(eltfile) results.append([bank.bank['name'], release, fformat, ','.join(atypes), ','.join(atags), ','.join(afiles)]) print(tabulate(results, headers="firstrow", tablefmt="grid")) sys.exit(0) if options.check: if not options.bank: print("Bank name is missing") sys.exit(1) bank = Bank(options.bank, no_log=True) print(options.bank+" check: "+str(bank.check())+"\n") sys.exit(0) if options.status: if options.bank: bank = Bank(options.bank, no_log=True) info = bank.get_bank_release_info(full=True) print(tabulate(info['info'], headers='firstrow', tablefmt='psql')) print(tabulate(info['prod'], headers='firstrow', tablefmt='psql')) # do we have some pending release(s) if 'pend' in info and len(info['pend']) > 1: print(tabulate(info['pend'], headers='firstrow', tablefmt='psql')) else: banks = Bank.list() # Headers of output table banks_list = [["Name", "Type(s)", "Release", "Visibility"]] for bank in sorted(banks, key=lambda k: k['name']): bank = Bank(bank['name'], no_log=True) banks_list.append(bank.get_bank_release_info()['info']) print(tabulate(banks_list, headers="firstrow", tablefmt="psql")) sys.exit(0) if options.statusko: banks = Bank.list() banks_list = [["Name", "Type(s)", "Release", "Visibility"]] for bank in sorted(banks, key=lambda k: k['name']): try: bank = Bank(bank['name'], no_log=True) bank.load_session(UpdateWorkflow.FLOW) if bank.session is not None: if bank.use_last_session and not bank.session.get_status(Workflow.FLOW_OVER): wf_status = bank.session.get('workflow_status') if wf_status is None or not wf_status: banks_list.append(bank.get_bank_release_info()['info']) except Exception as e: print(str(e)) print(tabulate(banks_list, headers="firstrow", tablefmt="psql")) if options.update: if not options.bank: print("Bank name is missing") sys.exit(1) banks = options.bank.split(',') gres = True for bank in banks: options.bank = bank bmaj = Bank(bank, options) print('Log file: '+bmaj.config.log_file) check_status = bmaj.check() if not check_status: print('Skip bank ' + options.bank + ': wrong config') gres = False continue res = bmaj.update(depends=True) if not res: gres = False Notify.notifyBankAction(bmaj) if not gres: sys.exit(1) if options.freeze: if not options.bank: print("Bank name is missing") sys.exit(1) if not options.release: print("Bank release is missing") sys.exit(1) bmaj = Bank(options.bank, options) res = bmaj.freeze(options.release) if not res: sys.exit(1) if options.unfreeze: if not options.bank: print("Bank name is missing") sys.exit(1) if not options.release: print("Bank release is missing") sys.exit(1) bmaj = Bank(options.bank, options) res = bmaj.unfreeze(options.release) if not res: sys.exit(1) if options.remove or options.removeall: if not options.bank: print("Bank name is missing") sys.exit(1) if options.remove and not options.release: print("Bank release is missing") sys.exit(1) if options.removeall: bmaj = Bank(options.bank, options, no_log=True) print('Log file: '+bmaj.config.log_file) res = bmaj.removeAll(options.force) else: bmaj = Bank(options.bank, options) print('Log file: '+bmaj.config.log_file) res = bmaj.remove(options.release) Notify.notifyBankAction(bmaj) if not res: sys.exit(1) if options.removepending: if not options.bank: print("Bank name is missing") sys.exit(1) bmaj = Bank(options.bank, options, no_log=True) print('Log file: '+bmaj.config.log_file) res = bmaj.remove_pending(options.release) if not res: sys.exit(1) if options.unpublish: if not options.bank: print("Bank name is missing") sys.exit(1) bmaj = Bank(options.bank, options, no_log=True) bmaj.load_session() bmaj.unpublish() sys.exit(0) if options.publish: if not options.bank: print("Bank name or release is missing") sys.exit(1) bmaj = Bank(options.bank, options, no_log=True) print('Log file: '+bmaj.config.log_file) bmaj.load_session() bank = bmaj.bank session = None if options.get_option('release') is None: # Get latest prod release if len(bank['production'])>0: prod = bank['production'][len(bank['production'])-1] for s in bank['sessions']: if s['id'] == prod['session']: session = s break else: # Search production release matching release for prod in bank['production']: if prod['release'] == options.release or prod['prod_dir'] == options.release: # Search session related to this production release for s in bank['sessions']: if s['id'] == prod['session']: session = s break break if session is None: print("No production session could be found for this release") sys.exit(1) bmaj.session._session = session bmaj.publish() except Exception as e: print(str(e))