def command_fix(options, collection_name):
log_info('Fixing collection {}'.format(collection_name))
configuration = common.parse_File_Config(options)
collection = perform_scanner(configuration, collection_name)
for set in collection.sets:
if set.status == common.ROMset.SET_STATUS_BADNAME:
common.fix_ROM_set(set)
# Rescan collection and store results in chache.
command_scan(options, collection_name)
def command_deleteUnknown(options, collection_name):
log_info('Deleting Unknown SETs in collection {}'.format(collection_name))
configuration = common.parse_File_Config(options)
collection = perform_scanner(configuration, collection_name)
for set in collection.sets:
if set.status == common.ROMset.SET_STATUS_UNKNOWN:
print('Deleting {}'.format(set.basename))
os.remove(set.filename)
# Rescan collection and store results in chache.
command_scan(options, collection_name)
def process_arguments(args):
options = Options()
if args.verbose:
if args.verbose == 1:
common.change_log_level(common.LOG_VERB)
log_info('Verbosity level set to VERBOSE')
elif args.verbose == 2:
common.change_log_level(common.LOG_DEBUG)
log_info('Verbosity level set to DEBUG')
if args.dryRun:
__prog_option_dry_run = 1
return options
def run(self):
queue = self.queue
log_info('Builder queue for {} started'.format(self.builder))
while True:
scheduled_job = queue.get()
if not scheduled_job:
break
self._process_job(scheduled_job)
queue.task_done()
log_info('Builder queue for {} stopped'.format(self.builder))
def _process_job(self, scheduled_job: _BuildScheduledJob):
builder = self.builder
job = scheduled_job.job
success = True
log_info(f'{job} build started on {builder}')
try:
builder.build(job)
msg = f'{job} build on {builder}: succeed'
log_info(msg)
except Exception as exception: # pylint: disable=broad-except
msg = f'{job} build on {builder} failed: {str(exception)}'
log_error(msg)
success = False
scheduled_job.build_done(success, msg)
def command_scanall(options):
log_info('Scanning all collections')
configuration = common.parse_File_Config(options)
# Scan collection by collection.
stats_list = []
for collection_name in configuration.collections:
collection = perform_scanner(configuration, collection_name)
# Save scanner results for later.
scan_FN = options.data_dir_FN.pjoin(collection.name + '_scan.bin')
print('Saving scanner results in "{}"'.format(scan_FN.getPath()))
f = open(scan_FN.getPath(), 'wb')
pickle.dump(collection, f)
f.close()
def command_listIssues(options, collection_name):
log_info('List collection scanned ROMs with issues')
# Load collection scanner data.
scan_FN = options.data_dir_FN.pjoin(collection_name + '_scan.bin')
if not scan_FN.exists():
print('Not found {}'.format(scan_FN.getPath()))
print('Exiting')
sys.exit(1)
print('Loading scanner results in "{}"'.format(scan_FN.getPath()))
f = open(scan_FN.getPath(), 'rb')
collection = pickle.load(f)
f.close()
# Print scanner results (long list)
print('\n=== Scanner long list ===')
for set in collection.sets:
if set.status == common.ROMset.SET_STATUS_GOOD: continue
log_info('\033[91mSET\033[0m {} "{}"'.format(set.status, set.basename))
for rom in set.rom_list:
if rom['status'] == common.ROMset.ROM_STATUS_BADNAME:
log_info('ROM {} "{}" -> "{}"'.format(rom['status'],
rom['name'],
rom['correct_name']))
else:
log_info('ROM {} "{}"'.format(rom['status'], rom['name']))
def run(self):
queue = self.queue
log_info('Job queue started')
while True:
scheduled_job = queue.get()
if not scheduled_job:
queue.task_done()
break
job = scheduled_job.job
success = scheduled_job.success
feedback = '\n'.join(scheduled_job.feedback_msgs)
self._process_build_done(job, success, feedback)
queue.task_done()
log_info('Job queue stopped')
def command_statusall(options):
log_info('View all collections scan results')
configuration = common.parse_File_Config(options)
stats_list = []
for collection_name in configuration.collections:
# Load collection scanner data.
scan_FN = options.data_dir_FN.pjoin(collection_name + '_scan.bin')
if not scan_FN.exists():
print('Not found {}'.format(scan_FN.getPath()))
print('Exiting')
sys.exit(1)
print('Loading scanner results in "{}"'.format(scan_FN.getPath()))
f = open(scan_FN.getPath(), 'rb')
collection = pickle.load(f)
f.close()
stats = common.get_collection_statistics(collection)
stats_list.append(stats)
# Print results.
table_str = [
['left', 'left', 'left', 'left', 'left', 'left', 'left', 'left'],
[
'Collection', 'DAT SETs', 'Total ROMs', 'Have ROMs',
'BadName ROMs', 'Miss ROMs', 'Unknown ROMs', 'Error files'
],
]
for stats in stats_list:
table_str.append([
str(stats['name']),
str(stats['total_DAT']),
str(stats['total']),
str(stats['have']),
str(stats['badname']),
str(stats['missing']),
str(stats['unknown']),
str(stats['error']),
])
table_text = common.text_render_table(table_str)
print('')
for line in table_text:
print(line)
def perform_scanner(configuration, collection_name):
log_info('***** Scanning collection {} *****'.format(collection_name))
if collection_name not in configuration.collections:
log_error(
'Collection "{}" not found in the configuration file.'.format(
collection_name))
sys.exit(1)
collection_conf = configuration.collections[collection_name]
# Load DAT file.
DAT_dir_FN = FileName(configuration.common_opts['NoIntro_DAT_dir'])
DAT_FN = DAT_dir_FN.pjoin(collection_conf['DAT'])
DAT = common.load_XML_DAT_file(DAT_FN)
# Scan files in ROM_dir.
collection = ROMcollection(collection_conf)
collection.scan_files_in_dir()
collection.process_files(DAT)
return collection
def command_scan(options, collection_name):
log_info('Scanning collection')
configuration = common.parse_File_Config(options)
collection = perform_scanner(configuration, collection_name)
# Save scanner results for later.
scan_FN = options.data_dir_FN.pjoin(collection.name + '_scan.bin')
print('Saving scanner results in "{}"'.format(scan_FN.getPath()))
f = open(scan_FN.getPath(), 'wb')
pickle.dump(collection, f)
f.close()
# Print scanner summary.
stats = common.get_collection_statistics(collection)
print('\n=== Scanner summary for collection "{}" ==='.format(
collection_name))
print('Total SETs in DAT {:5,}'.format(stats['total_DAT']))
print('Total SETs {:5,}'.format(stats['total']))
print('Have SETs {:5,}'.format(stats['have']))
print('Badname SETs {:5,}'.format(stats['badname']))
print('Miss SETs {:5,}'.format(stats['missing']))
print('Unknown SETs {:5,}'.format(stats['unknown']))
print('Error SETs {:5,}'.format(stats['error']))
def command_listcollections(options):
log_info('Listing ROM Collections in the configuration file')
# Read the configuration file.
configuration = common.parse_File_Config(options)
# pprint.pprint(configuration.collections)
# Print the ROM Collections in the configuration file.
# Use the table printing engine from AML/AEL.
table_str = [
['left', 'left', 'left'],
['Name', 'DAT file', 'ROM dir'],
]
for key in configuration.collections:
collection = configuration.collections[key]
table_str.append(
[collection['name'], collection['DAT'], collection['ROM_dir']])
# Print table
table_text = common.text_render_table(table_str)
print('')
for line in table_text:
print(line)
def command_status(options, collection_name):
log_info('View collection scan results')
# Load collection scanner data.
scan_FN = options.data_dir_FN.pjoin(collection_name + '_scan.bin')
if not scan_FN.exists():
print('Not found {}'.format(scan_FN.getPath()))
print('Exiting')
sys.exit(1)
print('Loading scanner results in "{}"'.format(scan_FN.getPath()))
f = open(scan_FN.getPath(), 'rb')
collection = pickle.load(f)
f.close()
# Print scanner summary.
stats = common.get_collection_statistics(collection)
print('\n=== Scanner summary for collection "{}" ==='.format(
collection_name))
print('Total SETs in DAT {:5,}'.format(stats['total_DAT']))
print('Total SETs {:5,}'.format(stats['total']))
print('Have SETs {:5,}'.format(stats['have']))
print('Badname SETs {:5,}'.format(stats['badname']))
print('Miss SETs {:5,}'.format(stats['missing']))
print('Unknown SETs {:5,}'.format(stats['unknown']))
print('Error SETs {:5,}'.format(stats['error']))
def _process_build_done(self, job: BuildJob, success: bool, feedback: str):
if not success:
job.notify_result(False, feedback)
return
if not job.do_upload:
job.notify_result(True, f'{job}: upload skipped')
return
modified_repos = []
# begin changes in modified repositories (this gets the lock on those)
for arch in job.archs:
repo = self.repos[job.upload_repo][arch]
repo.begin()
modified_repos.append(repo)
try:
# Update repositories
manifest = job.merge_manifests()
for repo in modified_repos:
repo.add(manifest)
except Exception as exception: # pylint: disable=broad-except
# Rollback changes in modified repositories
for repo in modified_repos:
repo.rollback()
job.notify_result(False, str(exception))
log_error(f'{job} upload cancelled')
return
# Commit changes in modified repositories
for repo in modified_repos:
repo.commit()
log_info(f'{job} uploaded on {repo}')
job.notify_result(True)
def command_listStuff(options, collection_name, list_type):
log_info('List collection scanned ROMs with issues')
# Load collection scanner data.
scan_FN = options.data_dir_FN.pjoin(collection_name + '_scan.bin')
if not scan_FN.exists():
print('Not found {}'.format(scan_FN.getPath()))
print('Exiting')
sys.exit(1)
print('Loading scanner results in "{}"'.format(scan_FN.getPath()))
f = open(scan_FN.getPath(), 'rb')
collection = pickle.load(f)
f.close()
# Print scanner results (long list)
print('\n=== Scanner long list ===')
num_items = 0
for set in collection.sets:
# Filter SET
if list_type == LIST_BADNAME:
if set.status != common.ROMset.SET_STATUS_BADNAME: continue
elif list_type == LIST_MISSING:
if set.status != common.ROMset.SET_STATUS_MISSING: continue
elif list_type == LIST_UNKNOWN:
if set.status != common.ROMset.SET_STATUS_UNKNOWN: continue
elif list_type == LIST_ERROR:
if set.status != common.ROMset.SET_STATUS_ERROR: continue
else:
raise TypeError('Wrong type. Logical error.')
log_info('\033[91mSET\033[0m {} "{}"'.format(set.status, set.basename))
num_items += 1
for rom in set.rom_list:
if rom['status'] == common.ROMset.ROM_STATUS_BADNAME:
log_info('ROM {} "{}" -> "{}"'.format(rom['status'],
rom['name'],
rom['correct_name']))
else:
log_info('ROM {} "{}"'.format(rom['status'], rom['name']))
print('\nListed {} items.'.format(num_items))
def ping(self):
while self.status:
try:
time.sleep(60)
# 尝试向redis-server发一条消息
if not self.conn.ping():
log_info("redis连接丢失，重新获取连接")
self.conn = redis.Redis(host=my_host,
port=my_port,
db=my_db,
password=my_pw)
# self.conn = self.myredis.getConnect()
# self.redis_sub = self.get_psubscribe()
else:
log_info("发送心跳成功")
log_info(self.conn)
except Exception as e:
log_error("redis连接异常，发送心跳失败，重新连接")
log_error(e)
def generate_buildjobs(req: BuildRequest) -> Iterator[BuildJob]:
"""
Generate the mmpack source packages from a build request
"""
log_info(f'making source packages for {req}...')
with TemporaryDirectory(prefix='mmpack-src') as tmpdir:
args = [
'mmpack-build',
'--outdir=' + tmpdir,
'--builddir=' + tmpdir + '/build',
'mksource',
'--git',
'--tag=' + req.fetch_refspec,
]
if req.srctar_make_opts.get('version_from_vcs', False):
args.append('--update-version-from-vcs')
if req.srctar_make_opts.get('only_modified', True):
args.append('--multiproject-only-modified')
args.append(req.url)
proc = Popen(args, stdout=PIPE, encoding='utf-8')
num_prj = 0
for line in proc.stdout:
fields = line.strip().split()
if len(fields) != 3:
break
job = BuildJob(req, fields[0], fields[1], fields[2])
num_prj += 1
log_info(f'... {job.prj_name} {job.version} {job.srchash}')
yield job
if proc.wait() != 0:
log_error(f'{args} failed')
else:
log_info('... Done' if num_prj else 'No mmpack packaging')
def run(self):
log_info("thread redis start....")
while self.status:
try:
self.redis_sub = self.get_psubscribe()
except Exception as e:
log_error("订阅失败，5秒后重试")
log_error(e)
time.sleep(5)
continue
break
while self.status:
try:
msg = self.redis_sub.parse_response(block=False, timeout=60)
log_info("收到订阅消息 %s" % msg)
if msg is not None and msg[0] != 'psubscribe':
# self.data_list.append(msg[3])
phoneList.append(msg[3])
# print(phoneList.getData())
except Exception as e:
log_error("redis异常,重新检测连接")
log_error(e)
time.sleep(5)
log_info("thread redis end....")
def l(str):
common.log_info(str, gl.__logfile__)
--dryRun Don't modify any files, just print the operations to be done."""
)
# -----------------------------------------------------------------------------
# main function
# -----------------------------------------------------------------------------
print('\033[36mPython ROM Manager for No-Intro ROM sets\033[0m version ' +
common.PRM_VERSION)
# --- Initialise data and temp directories
# This is used to store the results of scans for later display. Use JSON to store data.
this_FN = FileName(__file__)
data_dir_FN = FileName(this_FN.getDir()).pjoin('data')
temp_dir_FN = FileName(this_FN.getDir()).pjoin('tmp')
log_info('Data dir "{}"'.format(data_dir_FN.getPath()))
log_info('Temp dir "{}"'.format(temp_dir_FN.getPath()))
# --- Command line parser
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', help='Bbe verbose', action='count')
parser.add_argument('--dryRun',
help='Do not modify any files',
action='store_true')
parser.add_argument('command', help='Main action to do', nargs=1)
parser.add_argument('collection', help='ROM collection name', nargs='?')
args = parser.parse_args()
options = process_arguments(args)
options.data_dir_FN = data_dir_FN
options.temp_dir_FN = temp_dir_FN
# pprint.pprint(args)
