def command_fix(options, collection_name): log_info('Fixing collection {}'.format(collection_name)) configuration = common.parse_File_Config(options) collection = perform_scanner(configuration, collection_name) for set in collection.sets: if set.status == common.ROMset.SET_STATUS_BADNAME: common.fix_ROM_set(set) # Rescan collection and store results in chache. command_scan(options, collection_name)
def command_deleteUnknown(options, collection_name): log_info('Deleting Unknown SETs in collection {}'.format(collection_name)) configuration = common.parse_File_Config(options) collection = perform_scanner(configuration, collection_name) for set in collection.sets: if set.status == common.ROMset.SET_STATUS_UNKNOWN: print('Deleting {}'.format(set.basename)) os.remove(set.filename) # Rescan collection and store results in chache. command_scan(options, collection_name)
def process_arguments(args): options = Options() if args.verbose: if args.verbose == 1: common.change_log_level(common.LOG_VERB) log_info('Verbosity level set to VERBOSE') elif args.verbose == 2: common.change_log_level(common.LOG_DEBUG) log_info('Verbosity level set to DEBUG') if args.dryRun: __prog_option_dry_run = 1 return options
def run(self): queue = self.queue log_info('Builder queue for {} started'.format(self.builder)) while True: scheduled_job = queue.get() if not scheduled_job: break self._process_job(scheduled_job) queue.task_done() log_info('Builder queue for {} stopped'.format(self.builder))
def _process_job(self, scheduled_job: _BuildScheduledJob): builder = self.builder job = scheduled_job.job success = True log_info(f'{job} build started on {builder}') try: builder.build(job) msg = f'{job} build on {builder}: succeed' log_info(msg) except Exception as exception: # pylint: disable=broad-except msg = f'{job} build on {builder} failed: {str(exception)}' log_error(msg) success = False scheduled_job.build_done(success, msg)
def command_scanall(options): log_info('Scanning all collections') configuration = common.parse_File_Config(options) # Scan collection by collection. stats_list = [] for collection_name in configuration.collections: collection = perform_scanner(configuration, collection_name) # Save scanner results for later. scan_FN = options.data_dir_FN.pjoin(collection.name + '_scan.bin') print('Saving scanner results in "{}"'.format(scan_FN.getPath())) f = open(scan_FN.getPath(), 'wb') pickle.dump(collection, f) f.close()
def command_listIssues(options, collection_name): log_info('List collection scanned ROMs with issues') # Load collection scanner data. scan_FN = options.data_dir_FN.pjoin(collection_name + '_scan.bin') if not scan_FN.exists(): print('Not found {}'.format(scan_FN.getPath())) print('Exiting') sys.exit(1) print('Loading scanner results in "{}"'.format(scan_FN.getPath())) f = open(scan_FN.getPath(), 'rb') collection = pickle.load(f) f.close() # Print scanner results (long list) print('\n=== Scanner long list ===') for set in collection.sets: if set.status == common.ROMset.SET_STATUS_GOOD: continue log_info('\033[91mSET\033[0m {} "{}"'.format(set.status, set.basename)) for rom in set.rom_list: if rom['status'] == common.ROMset.ROM_STATUS_BADNAME: log_info('ROM {} "{}" -> "{}"'.format(rom['status'], rom['name'], rom['correct_name'])) else: log_info('ROM {} "{}"'.format(rom['status'], rom['name']))
def run(self): queue = self.queue log_info('Job queue started') while True: scheduled_job = queue.get() if not scheduled_job: queue.task_done() break job = scheduled_job.job success = scheduled_job.success feedback = '\n'.join(scheduled_job.feedback_msgs) self._process_build_done(job, success, feedback) queue.task_done() log_info('Job queue stopped')
def command_statusall(options): log_info('View all collections scan results') configuration = common.parse_File_Config(options) stats_list = [] for collection_name in configuration.collections: # Load collection scanner data. scan_FN = options.data_dir_FN.pjoin(collection_name + '_scan.bin') if not scan_FN.exists(): print('Not found {}'.format(scan_FN.getPath())) print('Exiting') sys.exit(1) print('Loading scanner results in "{}"'.format(scan_FN.getPath())) f = open(scan_FN.getPath(), 'rb') collection = pickle.load(f) f.close() stats = common.get_collection_statistics(collection) stats_list.append(stats) # Print results. table_str = [ ['left', 'left', 'left', 'left', 'left', 'left', 'left', 'left'], [ 'Collection', 'DAT SETs', 'Total ROMs', 'Have ROMs', 'BadName ROMs', 'Miss ROMs', 'Unknown ROMs', 'Error files' ], ] for stats in stats_list: table_str.append([ str(stats['name']), str(stats['total_DAT']), str(stats['total']), str(stats['have']), str(stats['badname']), str(stats['missing']), str(stats['unknown']), str(stats['error']), ]) table_text = common.text_render_table(table_str) print('') for line in table_text: print(line)
def perform_scanner(configuration, collection_name): log_info('***** Scanning collection {} *****'.format(collection_name)) if collection_name not in configuration.collections: log_error( 'Collection "{}" not found in the configuration file.'.format( collection_name)) sys.exit(1) collection_conf = configuration.collections[collection_name] # Load DAT file. DAT_dir_FN = FileName(configuration.common_opts['NoIntro_DAT_dir']) DAT_FN = DAT_dir_FN.pjoin(collection_conf['DAT']) DAT = common.load_XML_DAT_file(DAT_FN) # Scan files in ROM_dir. collection = ROMcollection(collection_conf) collection.scan_files_in_dir() collection.process_files(DAT) return collection
def command_scan(options, collection_name): log_info('Scanning collection') configuration = common.parse_File_Config(options) collection = perform_scanner(configuration, collection_name) # Save scanner results for later. scan_FN = options.data_dir_FN.pjoin(collection.name + '_scan.bin') print('Saving scanner results in "{}"'.format(scan_FN.getPath())) f = open(scan_FN.getPath(), 'wb') pickle.dump(collection, f) f.close() # Print scanner summary. stats = common.get_collection_statistics(collection) print('\n=== Scanner summary for collection "{}" ==='.format( collection_name)) print('Total SETs in DAT {:5,}'.format(stats['total_DAT'])) print('Total SETs {:5,}'.format(stats['total'])) print('Have SETs {:5,}'.format(stats['have'])) print('Badname SETs {:5,}'.format(stats['badname'])) print('Miss SETs {:5,}'.format(stats['missing'])) print('Unknown SETs {:5,}'.format(stats['unknown'])) print('Error SETs {:5,}'.format(stats['error']))
def command_listcollections(options): log_info('Listing ROM Collections in the configuration file') # Read the configuration file. configuration = common.parse_File_Config(options) # pprint.pprint(configuration.collections) # Print the ROM Collections in the configuration file. # Use the table printing engine from AML/AEL. table_str = [ ['left', 'left', 'left'], ['Name', 'DAT file', 'ROM dir'], ] for key in configuration.collections: collection = configuration.collections[key] table_str.append( [collection['name'], collection['DAT'], collection['ROM_dir']]) # Print table table_text = common.text_render_table(table_str) print('') for line in table_text: print(line)
def command_status(options, collection_name): log_info('View collection scan results') # Load collection scanner data. scan_FN = options.data_dir_FN.pjoin(collection_name + '_scan.bin') if not scan_FN.exists(): print('Not found {}'.format(scan_FN.getPath())) print('Exiting') sys.exit(1) print('Loading scanner results in "{}"'.format(scan_FN.getPath())) f = open(scan_FN.getPath(), 'rb') collection = pickle.load(f) f.close() # Print scanner summary. stats = common.get_collection_statistics(collection) print('\n=== Scanner summary for collection "{}" ==='.format( collection_name)) print('Total SETs in DAT {:5,}'.format(stats['total_DAT'])) print('Total SETs {:5,}'.format(stats['total'])) print('Have SETs {:5,}'.format(stats['have'])) print('Badname SETs {:5,}'.format(stats['badname'])) print('Miss SETs {:5,}'.format(stats['missing'])) print('Unknown SETs {:5,}'.format(stats['unknown'])) print('Error SETs {:5,}'.format(stats['error']))
def _process_build_done(self, job: BuildJob, success: bool, feedback: str): if not success: job.notify_result(False, feedback) return if not job.do_upload: job.notify_result(True, f'{job}: upload skipped') return modified_repos = [] # begin changes in modified repositories (this gets the lock on those) for arch in job.archs: repo = self.repos[job.upload_repo][arch] repo.begin() modified_repos.append(repo) try: # Update repositories manifest = job.merge_manifests() for repo in modified_repos: repo.add(manifest) except Exception as exception: # pylint: disable=broad-except # Rollback changes in modified repositories for repo in modified_repos: repo.rollback() job.notify_result(False, str(exception)) log_error(f'{job} upload cancelled') return # Commit changes in modified repositories for repo in modified_repos: repo.commit() log_info(f'{job} uploaded on {repo}') job.notify_result(True)
def command_listStuff(options, collection_name, list_type): log_info('List collection scanned ROMs with issues') # Load collection scanner data. scan_FN = options.data_dir_FN.pjoin(collection_name + '_scan.bin') if not scan_FN.exists(): print('Not found {}'.format(scan_FN.getPath())) print('Exiting') sys.exit(1) print('Loading scanner results in "{}"'.format(scan_FN.getPath())) f = open(scan_FN.getPath(), 'rb') collection = pickle.load(f) f.close() # Print scanner results (long list) print('\n=== Scanner long list ===') num_items = 0 for set in collection.sets: # Filter SET if list_type == LIST_BADNAME: if set.status != common.ROMset.SET_STATUS_BADNAME: continue elif list_type == LIST_MISSING: if set.status != common.ROMset.SET_STATUS_MISSING: continue elif list_type == LIST_UNKNOWN: if set.status != common.ROMset.SET_STATUS_UNKNOWN: continue elif list_type == LIST_ERROR: if set.status != common.ROMset.SET_STATUS_ERROR: continue else: raise TypeError('Wrong type. Logical error.') log_info('\033[91mSET\033[0m {} "{}"'.format(set.status, set.basename)) num_items += 1 for rom in set.rom_list: if rom['status'] == common.ROMset.ROM_STATUS_BADNAME: log_info('ROM {} "{}" -> "{}"'.format(rom['status'], rom['name'], rom['correct_name'])) else: log_info('ROM {} "{}"'.format(rom['status'], rom['name'])) print('\nListed {} items.'.format(num_items))
def ping(self): while self.status: try: time.sleep(60) # 尝试向redis-server发一条消息 if not self.conn.ping(): log_info("redis连接丢失,重新获取连接") self.conn = redis.Redis(host=my_host, port=my_port, db=my_db, password=my_pw) # self.conn = self.myredis.getConnect() # self.redis_sub = self.get_psubscribe() else: log_info("发送心跳成功") log_info(self.conn) except Exception as e: log_error("redis连接异常,发送心跳失败,重新连接") log_error(e)
def generate_buildjobs(req: BuildRequest) -> Iterator[BuildJob]: """ Generate the mmpack source packages from a build request """ log_info(f'making source packages for {req}...') with TemporaryDirectory(prefix='mmpack-src') as tmpdir: args = [ 'mmpack-build', '--outdir=' + tmpdir, '--builddir=' + tmpdir + '/build', 'mksource', '--git', '--tag=' + req.fetch_refspec, ] if req.srctar_make_opts.get('version_from_vcs', False): args.append('--update-version-from-vcs') if req.srctar_make_opts.get('only_modified', True): args.append('--multiproject-only-modified') args.append(req.url) proc = Popen(args, stdout=PIPE, encoding='utf-8') num_prj = 0 for line in proc.stdout: fields = line.strip().split() if len(fields) != 3: break job = BuildJob(req, fields[0], fields[1], fields[2]) num_prj += 1 log_info(f'... {job.prj_name} {job.version} {job.srchash}') yield job if proc.wait() != 0: log_error(f'{args} failed') else: log_info('... Done' if num_prj else 'No mmpack packaging')
def run(self): log_info("thread redis start....") while self.status: try: self.redis_sub = self.get_psubscribe() except Exception as e: log_error("订阅失败,5秒后重试") log_error(e) time.sleep(5) continue break while self.status: try: msg = self.redis_sub.parse_response(block=False, timeout=60) log_info("收到订阅消息 %s" % msg) if msg is not None and msg[0] != 'psubscribe': # self.data_list.append(msg[3]) phoneList.append(msg[3]) # print(phoneList.getData()) except Exception as e: log_error("redis异常,重新检测连接") log_error(e) time.sleep(5) log_info("thread redis end....")
def l(str): common.log_info(str, gl.__logfile__)
--dryRun Don't modify any files, just print the operations to be done.""" ) # ----------------------------------------------------------------------------- # main function # ----------------------------------------------------------------------------- print('\033[36mPython ROM Manager for No-Intro ROM sets\033[0m version ' + common.PRM_VERSION) # --- Initialise data and temp directories # This is used to store the results of scans for later display. Use JSON to store data. this_FN = FileName(__file__) data_dir_FN = FileName(this_FN.getDir()).pjoin('data') temp_dir_FN = FileName(this_FN.getDir()).pjoin('tmp') log_info('Data dir "{}"'.format(data_dir_FN.getPath())) log_info('Temp dir "{}"'.format(temp_dir_FN.getPath())) # --- Command line parser parser = argparse.ArgumentParser() parser.add_argument('-v', '--verbose', help='Bbe verbose', action='count') parser.add_argument('--dryRun', help='Do not modify any files', action='store_true') parser.add_argument('command', help='Main action to do', nargs=1) parser.add_argument('collection', help='ROM collection name', nargs='?') args = parser.parse_args() options = process_arguments(args) options.data_dir_FN = data_dir_FN options.temp_dir_FN = temp_dir_FN # pprint.pprint(args)