class Options:
def __init__(self, name, defaults):
load_paths = list(reversed([os.path.join(directory, '%s.cfg' % name)
for directory
in load_config_paths(name)]))
self.save_path = os.path.join(save_config_path(name), '%s.cfg' % name)
self.config = RawConfigParser(defaults)
self.paths = self.config.read(load_paths)
self.section = name if self.config.has_section(name) else 'DEFAULT'
def get(self, option):
try:
return self.config.get(self.section, option)
except NoOptionError:
return None
def set(self, option, value):
if value is not None:
self.config.set(self.section, option, value)
else:
self.config.remove_option(self.section, option)
defaults = self.config._defaults
self.config._defaults = None
with open(self.save_path, 'w') as save_file:
self.config.write(save_file)
self.config._defaults = defaults
class Options:
def __init__(self, name, defaults):
load_paths = list(
reversed([
os.path.join(directory, '%s.cfg' % name)
for directory in load_config_paths(name)
]))
self.save_path = os.path.join(save_config_path(name), '%s.cfg' % name)
self.config = RawConfigParser(defaults)
self.paths = self.config.read(load_paths)
self.section = name if self.config.has_section(name) else 'DEFAULT'
def get(self, option):
try:
return self.config.get(self.section, option)
except NoOptionError:
return None
def set(self, option, value):
if value is not None:
self.config.set(self.section, option, value)
else:
self.config.remove_option(self.section, option)
defaults = self.config._defaults
self.config._defaults = None
with open(self.save_path, 'w') as save_file:
self.config.write(save_file)
self.config._defaults = defaults
class TestGithubService(TestCase):
def setUp(self):
self.config = RawConfigParser()
self.config.interactive = False
self.config.add_section('general')
self.config.add_section('mygithub')
self.config.set('mygithub', 'service', 'github')
self.config.set('mygithub', 'github.login', 'tintin')
self.config.set('mygithub', 'github.username', 'milou')
self.config.set('mygithub', 'github.password', 't0ps3cr3t')
self.service_config = ServiceConfig(GithubService.CONFIG_PREFIX,
self.config, 'mygithub')
def test_token_authorization_header(self):
self.config.remove_option('mygithub', 'github.password')
self.config.set('mygithub', 'github.token',
'@oracle:eval:echo 1234567890ABCDEF')
service = GithubService(self.config, 'general', 'mygithub')
self.assertEqual(service.client.session.headers['Authorization'],
"token 1234567890ABCDEF")
def test_default_host(self):
""" Check that if github.host is not set, we default to github.com """
service = GithubService(self.config, 'general', 'mygithub')
self.assertEquals("github.com", service.host)
def test_overwrite_host(self):
""" Check that if github.host is set, we use its value as host """
self.config.set('mygithub', 'github.host', 'github.example.com')
service = GithubService(self.config, 'general', 'mygithub')
self.assertEquals("github.example.com", service.host)
def test_keyring_service(self):
""" Checks that the keyring service name """
keyring_service = GithubService.get_keyring_service(
self.service_config)
self.assertEquals("github://[email protected]/milou", keyring_service)
def test_keyring_service_host(self):
""" Checks that the keyring key depends on the github host. """
self.config.set('mygithub', 'github.host', 'github.example.com')
keyring_service = GithubService.get_keyring_service(
self.service_config)
self.assertEquals("github://[email protected]/milou",
keyring_service)
def test_get_repository_from_issue_url__issue(self):
issue = dict(repos_url="https://github.com/foo/bar")
repository = GithubService.get_repository_from_issue(issue)
self.assertEquals("foo/bar", repository)
def test_get_repository_from_issue_url__pull_request(self):
issue = dict(repos_url="https://github.com/foo/bar")
repository = GithubService.get_repository_from_issue(issue)
self.assertEquals("foo/bar", repository)
def test_get_repository_from_issue__enterprise_github(self):
issue = dict(repos_url="https://github.acme.biz/foo/bar")
repository = GithubService.get_repository_from_issue(issue)
self.assertEquals("foo/bar", repository)
class TestGithubService(TestCase):
def setUp(self):
self.config = RawConfigParser()
self.config.interactive = False
self.config.add_section('general')
self.config.add_section('mygithub')
self.config.set('mygithub', 'service', 'github')
self.config.set('mygithub', 'github.login', 'tintin')
self.config.set('mygithub', 'github.username', 'milou')
self.config.set('mygithub', 'github.password', 't0ps3cr3t')
self.service_config = ServiceConfig(
GithubService.CONFIG_PREFIX, self.config, 'mygithub')
def test_token_authorization_header(self):
self.config.remove_option('mygithub', 'github.password')
self.config.set('mygithub', 'github.token',
'@oracle:eval:echo 1234567890ABCDEF')
service = GithubService(self.config, 'general', 'mygithub')
self.assertEqual(service.client.session.headers['Authorization'],
"token 1234567890ABCDEF")
def test_default_host(self):
""" Check that if github.host is not set, we default to github.com """
service = GithubService(self.config, 'general', 'mygithub')
self.assertEquals("github.com", service.host)
def test_overwrite_host(self):
""" Check that if github.host is set, we use its value as host """
self.config.set('mygithub', 'github.host', 'github.example.com')
service = GithubService(self.config, 'general', 'mygithub')
self.assertEquals("github.example.com", service.host)
def test_keyring_service(self):
""" Checks that the keyring service name """
keyring_service = GithubService.get_keyring_service(self.service_config)
self.assertEquals("github://[email protected]/milou", keyring_service)
def test_keyring_service_host(self):
""" Checks that the keyring key depends on the github host. """
self.config.set('mygithub', 'github.host', 'github.example.com')
keyring_service = GithubService.get_keyring_service(self.service_config)
self.assertEquals("github://[email protected]/milou", keyring_service)
def test_get_repository_from_issue_url__issue(self):
issue = dict(repos_url="https://github.com/foo/bar")
repository = GithubService.get_repository_from_issue(issue)
self.assertEquals("foo/bar", repository)
def test_get_repository_from_issue_url__pull_request(self):
issue = dict(repos_url="https://github.com/foo/bar")
repository = GithubService.get_repository_from_issue(issue)
self.assertEquals("foo/bar", repository)
def test_get_repository_from_issue__enterprise_github(self):
issue = dict(repos_url="https://github.acme.biz/foo/bar")
repository = GithubService.get_repository_from_issue(issue)
self.assertEquals("foo/bar", repository)
def remove_and_save(self, option, section=DEFSECTION):
"""
Remove an option and then save it the config file
"""
if self.has_section(section):
RawConfigParser.remove_option(self, section, option)
if not self.remove_in_file(section, option):
return (_('Unable to save the config file'), 'Error')
return (_('Option %s deleted') % option, 'Info')
def remove_and_save(self, option, section=DEFSECTION):
"""
Remove an option and then save it the config file
"""
if self.has_section(section):
RawConfigParser.remove_option(self, section, option)
if not self.remove_in_file(section, option):
return ('Unable to save the config file', 'Error')
return ('Option %s deleted' % option, 'Info')
def remove_option(self, option):
"""
Removes an option (in ``section/key`` syntax),
thus will not be saved anymore
:param option: the option path
:type option: string
"""
splitvals = option.split('/')
section, key = "/".join(splitvals[:-1]), splitvals[-1]
RawConfigParser.remove_option(self, section, key)
def parse_config(self, filename):
#from ConfigParser import RawConfigParser
from configparser import RawConfigParser
import io
config = RawConfigParser()
config.readfp(io.open(filename, 'r', encoding='utf_8_sig'))
for s in config.sections():
port = int(config.get(s, 'port'))
config.remove_option(s, 'port')
xsize, ysize = [int(d) for d in config.get(s, 'size').split(",")]
config.remove_option(s, 'size')
x_off, y_off = [int(d) for d in config.get(s, 'offset').split(",")]
config.remove_option(s, 'offset')
self.offsets[s] = (x_off, y_off)
for device, offset in config.items(s):
x_off, y_off = [int(d) for d in offset.split(",")]
if device in self.offsets:
if (x_off, y_off) != self.offsets[device]:
raise RuntimeError(
"conflicting offsets for device %s" % device)
self.offsets[device] = (x_off, y_off)
if s in self.transtbl: self.transtbl[s].append(device)
else: self.transtbl[s] = [device]
if device in self.transtbl: self.transtbl[device].append(s)
else: self.transtbl[device] = [s]
self.add_virtual(s, xsize, ysize, port)
def edit_config(filename, settings, dry_run=False):
"""Edit a configuration file to include `settings`
`settings` is a dictionary of dictionaries or ``None`` values, keyed by
command/section name. A ``None`` value means to delete the entire section,
while a dictionary lists settings to be changed or deleted in that section.
A setting of ``None`` means to delete that setting.
"""
from configparser import RawConfigParser
log.debug("Reading configuration from %s", filename)
opts = RawConfigParser()
opts.read([filename])
for section, options in list(settings.items()):
if options is None:
log.info("Deleting section [%s] from %s", section, filename)
opts.remove_section(section)
else:
if not opts.has_section(section):
log.debug("Adding new section [%s] to %s", section, filename)
opts.add_section(section)
for option,value in list(options.items()):
if value is None:
log.debug("Deleting %s.%s from %s",
section, option, filename
)
opts.remove_option(section,option)
if not opts.options(section):
log.info("Deleting empty [%s] section from %s",
section, filename)
opts.remove_section(section)
else:
log.debug(
"Setting %s.%s to %r in %s",
section, option, value, filename
)
opts.set(section,option,value)
log.info("Writing %s", filename)
if not dry_run:
f = open(filename,'w'); opts.write(f); f.close()
def cleanup():
cfg_files = glob(CFGFILES)
for file in cfg_files:
parser = RawConfigParser(allow_no_value=True)
parser.optionxform = str
parser.read(unicode(file))
for section in parser:
if not section.startswith('plot:'):
if(section != 'DEFAULT'):
print('Invalid configuration section: %s:%s, skipping.' % (file, section))
continue
if not PLOTNAMEPATTERN.match(section.lstrip('plot:')):
print("Invalid plot name: '%s:%s' Plot names can contain only: [a-zA-Z0-9_+-]" % (file, section.lstrip('plot:')))
continue
if 'metric' not in parser[section] or\
'relativePath' not in parser[section] or\
'yTitle' not in parser[section]:
print('Plot missing required attributes: %s:%s, skipping.' % (file, section))
print('Required parameters: metric, relativePath, yTitle')
continue
parser.remove_option(section, 'runOffset')
parser.remove_option(section, 'relSystematic')
parser.remove_option(section, 'absSystematic')
parser.remove_option(section, 'yMin')
parser.remove_option(section, 'yMax')
parser.remove_option(section, 'yMin')
parser.remove_option(section, 'yMin')
if 'hTitle' in parser[section]:
parser[section]['plotTitle'] = parser[section]['hTitle']
parser.remove_option(section, 'hTitle')
with open(file, 'w') as configfile:
parser.write(configfile)
def edit_config(filename, settings, dry_run=False):
"""Edit a configuration file to include `settings`
`settings` is a dictionary of dictionaries or ``None`` values, keyed by
command/section name. A ``None`` value means to delete the entire section,
while a dictionary lists settings to be changed or deleted in that section.
A setting of ``None`` means to delete that setting.
"""
from configparser import RawConfigParser
log.debug("Reading configuration from %s", filename)
opts = RawConfigParser()
opts.read([filename])
for section, options in list(settings.items()):
if options is None:
log.info("Deleting section [%s] from %s", section, filename)
opts.remove_section(section)
else:
if not opts.has_section(section):
log.debug("Adding new section [%s] to %s", section, filename)
opts.add_section(section)
for option, value in list(options.items()):
if value is None:
log.debug("Deleting %s.%s from %s", section, option,
filename)
opts.remove_option(section, option)
if not opts.options(section):
log.info("Deleting empty [%s] section from %s",
section, filename)
opts.remove_section(section)
else:
log.debug("Setting %s.%s to %r in %s", section, option,
value, filename)
opts.set(section, option, value)
log.info("Writing %s", filename)
if not dry_run:
f = open(filename, 'w')
opts.write(f)
f.close()
class Config:
def __init__(self, path):
self.mutex = RLock()
self.path = path
self.config = RawConfigParser()
self.config.optionxform = str
try:
with open(path) as fp:
self.config.read_file(fp)
except:
pass
def set(self, section, key, value):
with self.mutex:
try:
self.config.set(section, key, value)
except NoSectionError:
self.config.add_section(section)
self.config.set(section, key, value)
self.save()
def get(self, section, key, fallback=_UNSET):
return self.config.get(section, key, fallback=fallback)
def getInt(self, section, key, fallback=_UNSET):
return self.config.getint(section, key, fallback=fallback)
def remove(self, section, key):
with self.mutex:
result = self.config.remove_option(section, key)
self.save()
def save(self):
with self.mutex:
with open(self.path, 'w') as configfile:
self.config.write(configfile)
def sections(self):
return self.config.sections()
def main(original_args=None):
positionals, args = split_args_in_optional_and_positional(
sys.argv[1:] if original_args is None else original_args)
if len(positionals[1:]) > 2:
warnings.warn(
"Giving multiple files on the command line will be deprecated, please use [bumpversion:file:...] in a config file.",
PendingDeprecationWarning)
parser1 = argparse.ArgumentParser(add_help=False)
parser1.add_argument('--verbose',
action='count',
default=0,
help='Print verbose logging to stderr',
required=False)
parser1.add_argument('--list',
action='store_true',
default=False,
help='List machine readable information',
required=False)
known_args, remaining_argv = parser1.parse_known_args(args)
logformatter = logging.Formatter('%(message)s')
if len(logger.handlers) == 0:
ch = logging.StreamHandler(sys.stderr)
ch.setFormatter(logformatter)
logger.addHandler(ch)
if len(logger_list.handlers) == 0:
ch2 = logging.StreamHandler(sys.stdout)
ch2.setFormatter(logformatter)
logger_list.addHandler(ch2)
if known_args.list:
logger_list.setLevel(1)
log_level = {
0: logging.WARNING,
1: logging.INFO,
2: logging.DEBUG,
}.get(known_args.verbose, logging.DEBUG)
logger.setLevel(log_level)
logger.debug("Starting {}".format(DESCRIPTION))
defaults = {}
vcs_info = {}
config = RawConfigParser('')
# don't transform keys to lowercase (which would be the default)
config.optionxform = lambda option: option
config.add_section('bumpversion')
# We need setup.py to get the major, minor versions
ver_sources = ['setup.py', 'plugin.json', 'VERSION']
ver_source = ver_file_check(ver_sources)
if ver_source is None:
message = "Could not read any of {} file".format(str(ver_sources))
logger.error(message)
sys.exit(2)
# We don't work with other configuration files except .bumpversion.cfg
config_file = '.bumpversion.cfg'
if not os.path.exists(config_file):
message = "Could not read {} file".format(config_file)
logger.error(message)
sys.exit(2)
part_configs = {}
files = []
logger.info("Reading config file {}:".format(config_file))
logger.info(io.open(config_file, 'rt', encoding='utf-8').read())
config.readfp(io.open(config_file, 'rt', encoding='utf-8'))
log_config = StringIO()
config.write(log_config)
if 'files' in dict(config.items("bumpversion")):
warnings.warn(
"'files =' configuration is will be deprecated, please use [bumpversion:file:...]",
PendingDeprecationWarning)
defaults.update(dict(config.items("bumpversion")))
for listvaluename in ("serialize", ):
try:
value = config.get("bumpversion", listvaluename)
defaults[listvaluename] = list(
filter(None, (x.strip() for x in value.splitlines())))
except NoOptionError:
pass # no default value then ;)
for boolvaluename in "dry_run":
try:
defaults[boolvaluename] = config.getboolean(
"bumpversion", boolvaluename)
except NoOptionError:
pass # no default value then ;)
for section_name in config.sections():
section_name_match = re.compile("^bumpversion:(file|part):(.+)").match(
section_name)
if not section_name_match:
continue
section_prefix, section_value = section_name_match.groups()
section_config = dict(config.items(section_name))
if section_prefix == "part":
ThisVersionPartConfiguration = NumericVersionPartConfiguration
if 'values' in section_config:
section_config['values'] = list(
filter(None,
(x.strip()
for x in section_config['values'].splitlines())))
ThisVersionPartConfiguration = ConfiguredVersionPartConfiguration
part_configs[section_value] = ThisVersionPartConfiguration(
**section_config)
elif section_prefix == "file":
filename = section_value
if 'serialize' in section_config:
section_config['serialize'] = list(
filter(
None,
(x.strip()
for x in section_config['serialize'].splitlines())))
section_config['part_configs'] = part_configs
if not 'parse' in section_config:
section_config['parse'] = defaults.get(
"parse", '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)')
if not 'serialize' in section_config:
section_config['serialize'] = defaults.get(
'serialize', [str('{major}.{minor}.{patch}')])
if not 'search' in section_config:
section_config['search'] = defaults.get(
"search", '{current_version}')
if not 'replace' in section_config:
section_config['replace'] = defaults.get(
"replace", '{new_version}')
files.append(
ConfiguredFile(filename, VersionConfig(**section_config)))
parser2 = argparse.ArgumentParser(prog='bumpversion',
add_help=False,
parents=[parser1])
parser2.set_defaults(**defaults)
parser2.add_argument('--parse',
metavar='REGEX',
help='Regex parsing the version string',
default=defaults.get(
"parse",
'(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)'))
parser2.add_argument('--serialize',
metavar='FORMAT',
action=DiscardDefaultIfSpecifiedAppendAction,
help='How to format what is parsed back to a version',
default=defaults.get(
"serialize", [str('{major}.{minor}.{patch}')]))
parser2.add_argument('--search',
metavar='SEARCH',
help='Template for complete string to search',
default=defaults.get("search", '{current_version}'))
parser2.add_argument('--replace',
metavar='REPLACE',
help='Template for complete string to replace',
default=defaults.get("replace", '{new_version}'))
known_args, remaining_argv = parser2.parse_known_args(args)
defaults.update(vars(known_args))
assert type(known_args.serialize) == list
context = dict(
list(time_context.items()) + list(prefixed_environ().items()) +
list(vcs_info.items()))
try:
vc = VersionConfig(
parse=known_args.parse,
serialize=known_args.serialize,
search=known_args.search,
replace=known_args.replace,
part_configs=part_configs,
)
except sre_constants.error as e:
sys.exit(1)
current_version = vc.parse(
known_args.current_version) if known_args.current_version else None
leave_config_ver = True
new_version = None
if len(positionals) > 0:
setup_version, zero_patch_setup_version = ConfiguredFile(
ver_source, vc).find(positionals[0])
compare = setup_version.compare(vc.order(), current_version)
for part in compare:
if part == positionals[0]:
continue
else:
leave_config_ver = leave_config_ver and compare[part]
try:
if leave_config_ver and current_version:
logger.info("Attempting to increment part '{}'".format(
positionals[0]))
new_version = current_version.bump(positionals[0], vc.order())
logger.info("Values are now: " +
keyvaluestring(new_version._values))
defaults['new_version'] = vc.serialize(new_version, context)
elif not leave_config_ver:
logger.info("Using Version from {}".format(ver_source))
defaults['new_version'] = vc.serialize(
zero_patch_setup_version, context)
new_version = zero_patch_setup_version
logger.info("Values are now: " +
keyvaluestring(setup_version._values))
except MissingValueForSerializationException as e:
logger.info("Opportunistic finding of new_version failed: " +
e.message)
except IncompleteVersionRepresenationException as e:
logger.info("Opportunistic finding of new_version failed: " +
e.message)
except KeyError as e:
logger.info("Opportunistic finding of new_version failed")
parser3 = argparse.ArgumentParser(
prog='bumpversion',
description=DESCRIPTION,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
conflict_handler='resolve',
parents=[parser2],
)
parser3.set_defaults(**defaults)
parser3.add_argument('--dry-run',
'-n',
action='store_true',
default=False,
help="Don't write any files, just pretend.")
file_names = []
if 'files' in defaults:
assert defaults['files'] != None
file_names = defaults['files'].split(' ')
parser3.add_argument('part', help='Part of the version to be bumped.')
parser3.add_argument('files',
metavar='file',
nargs='*',
help='Files to change',
default=file_names)
args = parser3.parse_args(remaining_argv + positionals)
if args.dry_run:
logger.info("Dry run active, won't touch any files.")
# make sure files exist and contain version string
# if leave_config_ver and new_version:
logger.info("Update info in {}".format(ver_source))
ConfiguredFile(ver_source, vc).replace(setup_version, new_version, context,
args.dry_run)
config.set('bumpversion', 'new_version', args.new_version)
for key, value in config.items('bumpversion'):
logger_list.info("{}={}".format(key, value))
config.remove_option('bumpversion', 'new_version')
config.set('bumpversion', 'current_version', args.new_version)
new_config = StringIO()
try:
write_to_config_file = not args.dry_run
logger.info("{} to config file {}:".format(
"Would write" if not write_to_config_file else "Writing",
config_file,
))
config.write(new_config)
logger.info(new_config.getvalue())
if write_to_config_file:
with io.open(config_file, 'wb') as f:
f.write(new_config.getvalue().encode('utf-8'))
except UnicodeEncodeError:
warnings.warn(
"Unable to write UTF-8 to config file, because of an old configparser version. "
"Update with `pip install --upgrade configparser`.")
def remove_option(self, option):
return _RawConfigParser.remove_option(self, self._default_section,
option)
class Config(object):
OUT_MKT_EDDN = 1
# OUT_MKT_BPC = 2 # No longer supported
OUT_MKT_TD = 4
OUT_MKT_CSV = 8
OUT_SHIP = 16
# OUT_SHIP_EDS = 16 # Replaced by OUT_SHIP
# OUT_SYS_FILE = 32 # No longer supported
# OUT_STAT = 64 # No longer available
# OUT_SHIP_CORIOLIS = 128 # Replaced by OUT_SHIP
OUT_STATION_ANY = OUT_MKT_EDDN|OUT_MKT_TD|OUT_MKT_CSV
# OUT_SYS_EDSM = 256 # Now a plugin
# OUT_SYS_AUTO = 512 # Now always automatic
OUT_MKT_MANUAL = 1024
OUT_SYS_EDDN = 2048
OUT_SYS_DELAY = 4096
if platform=='darwin':
def __init__(self):
self.app_dir = join(NSSearchPathForDirectoriesInDomains(NSApplicationSupportDirectory, NSUserDomainMask, True)[0], appname)
if not isdir(self.app_dir):
mkdir(self.app_dir)
self.plugin_dir = join(self.app_dir, 'plugins')
if not isdir(self.plugin_dir):
mkdir(self.plugin_dir)
self.internal_plugin_dir = getattr(sys, 'frozen', False) and normpath(join(dirname(sys.executable), pardir, 'Library', 'plugins')) or join(dirname(__file__), 'plugins')
self.default_journal_dir = join(NSSearchPathForDirectoriesInDomains(NSApplicationSupportDirectory, NSUserDomainMask, True)[0], 'Frontier Developments', 'Elite Dangerous')
self.home = expanduser('~')
self.respath = getattr(sys, 'frozen', False) and normpath(join(dirname(sys.executable), pardir, 'Resources')) or dirname(__file__)
if not getattr(sys, 'frozen', False):
# Don't use Python's settings if interactive
self.identifier = 'uk.org.marginal.%s' % appname.lower()
NSBundle.mainBundle().infoDictionary()['CFBundleIdentifier'] = self.identifier
else:
self.identifier = NSBundle.mainBundle().bundleIdentifier()
self.defaults = NSUserDefaults.standardUserDefaults()
self.settings = dict(self.defaults.persistentDomainForName_(self.identifier) or {}) # make writeable
# Check out_dir exists
if not self.get('outdir') or not isdir(self.get('outdir')):
self.set('outdir', NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, True)[0])
def get(self, key):
val = self.settings.get(key)
if val is None:
return None
elif isinstance(val, str):
return str(val)
elif hasattr(val, '__iter__'):
return list(val) # make writeable
else:
return None
def getint(self, key):
try:
return int(self.settings.get(key, 0)) # should already be int, but check by casting
except:
return 0
def set(self, key, val):
self.settings[key] = val
def delete(self, key):
self.settings.pop(key, None)
def save(self):
self.defaults.setPersistentDomain_forName_(self.settings, self.identifier)
self.defaults.synchronize()
def close(self):
self.save()
self.defaults = None
elif platform=='win32':
def __init__(self):
self.app_dir = join(KnownFolderPath(FOLDERID_LocalAppData), appname)
if not isdir(self.app_dir):
mkdir(self.app_dir)
self.plugin_dir = join(self.app_dir, 'plugins')
if not isdir(self.plugin_dir):
mkdir(self.plugin_dir)
self.internal_plugin_dir = join(dirname(getattr(sys, 'frozen', False) and sys.executable or __file__), u'plugins')
# expanduser in Python 2 on Windows doesn't handle non-ASCII - http://bugs.python.org/issue13207
self.home = KnownFolderPath(FOLDERID_Profile) or u'\\'
journaldir = KnownFolderPath(FOLDERID_SavedGames)
self.default_journal_dir = journaldir and join(journaldir, 'Frontier Developments', 'Elite Dangerous') or None
self.respath = dirname(getattr(sys, 'frozen', False) and sys.executable or __file__)
self.identifier = applongname
self.hkey = HKEY()
disposition = DWORD()
if RegCreateKeyEx(HKEY_CURRENT_USER, r'Software\EDD\EDD-EDMC', 0, None, 0, KEY_ALL_ACCESS, None, ctypes.byref(self.hkey), ctypes.byref(disposition)):
raise Exception()
# set WinSparkle defaults - https://github.com/vslavik/winsparkle/wiki/Registry-Settings
#sparklekey = HKEY()
#if not RegCreateKeyEx(self.hkey, 'WinSparkle', 0, None, 0, KEY_ALL_ACCESS, None, ctypes.byref(sparklekey), ctypes.byref(disposition)):
# if disposition.value == REG_CREATED_NEW_KEY:
# buf = ctypes.create_unicode_buffer('1')
# RegSetValueEx(sparklekey, 'CheckForUpdates', 0, 1, buf, len(buf)*2)
# buf = ctypes.create_unicode_buffer(str(update_interval))
# RegSetValueEx(sparklekey, 'UpdateInterval', 0, 1, buf, len(buf)*2)
# RegCloseKey(sparklekey)
if not self.get('outdir') or not isdir(self.get('outdir')):
self.set('outdir', KnownFolderPath(FOLDERID_Documents) or self.home)
def get(self, key):
typ = DWORD()
size = DWORD()
if RegQueryValueEx(self.hkey, key, 0, ctypes.byref(typ), None, ctypes.byref(size)) or typ.value not in [REG_SZ, REG_MULTI_SZ]:
return None
buf = ctypes.create_unicode_buffer(int(size.value / 2))
if RegQueryValueEx(self.hkey, key, 0, ctypes.byref(typ), buf, ctypes.byref(size)):
return None
elif typ.value == REG_MULTI_SZ:
return [x for x in ctypes.wstring_at(buf, len(buf)-2).split(u'\x00')]
else:
return str(buf.value)
def getint(self, key):
typ = DWORD()
size = DWORD(4)
val = DWORD()
if RegQueryValueEx(self.hkey, key, 0, ctypes.byref(typ), ctypes.byref(val), ctypes.byref(size)) or typ.value != REG_DWORD:
return 0
else:
return val.value
def set(self, key, val):
if isinstance(val, str):
buf = ctypes.create_unicode_buffer(val)
RegSetValueEx(self.hkey, key, 0, REG_SZ, buf, len(buf)*2)
elif isinstance(val, numbers.Integral):
RegSetValueEx(self.hkey, key, 0, REG_DWORD, ctypes.byref(DWORD(val)), 4)
elif hasattr(val, '__iter__'): # iterable
stringval = u'\x00'.join([str(x) or u' ' for x in val] + [u'']) # null terminated non-empty strings
buf = ctypes.create_unicode_buffer(stringval)
RegSetValueEx(self.hkey, key, 0, REG_MULTI_SZ, buf, len(buf)*2)
else:
raise NotImplementedError()
def delete(self, key):
RegDeleteValue(self.hkey, key)
def save(self):
pass # Redundant since registry keys are written immediately
def close(self):
RegCloseKey(self.hkey)
self.hkey = None
elif platform=='linux':
SECTION = 'config'
def __init__(self):
# http://standards.freedesktop.org/basedir-spec/latest/ar01s03.html
self.app_dir = join(getenv('XDG_DATA_HOME', expanduser('~/.local/share')), appname)
if not isdir(self.app_dir):
makedirs(self.app_dir)
self.plugin_dir = join(self.app_dir, 'plugins')
if not isdir(self.plugin_dir):
mkdir(self.plugin_dir)
self.internal_plugin_dir = join(dirname(__file__), 'plugins')
self.default_journal_dir = None
self.home = expanduser('~')
self.respath = dirname(__file__)
self.identifier = 'uk.org.marginal.%s' % appname.lower()
self.filename = join(getenv('XDG_CONFIG_HOME', expanduser('~/.config')), appname, '%s.ini' % appname)
if not isdir(dirname(self.filename)):
makedirs(dirname(self.filename))
self.config = RawConfigParser(comment_prefixes = ('#',))
try:
with codecs.open(self.filename, 'r') as h:
self.config.read_file(h)
except:
self.config.add_section(self.SECTION)
if not self.get('outdir') or not isdir(self.get('outdir')):
self.set('outdir', expanduser('~'))
def get(self, key):
try:
val = self.config.get(self.SECTION, key)
if u'\n' in val: # list
# ConfigParser drops the last entry if blank, so we add a spurious ';' entry in set() and remove it here
assert val.split('\n')[-1] == ';', val.split('\n')
return [self._unescape(x) for x in val.split(u'\n')[:-1]]
else:
return self._unescape(val)
except:
return None
def getint(self, key):
try:
return self.config.getint(self.SECTION, key)
except:
return 0
def set(self, key, val):
if isinstance(val, bool):
self.config.set(self.SECTION, key, val and '1' or '0')
elif isinstance(val, str) or isinstance(val, numbers.Integral):
self.config.set(self.SECTION, key, self._escape(val))
elif hasattr(val, '__iter__'): # iterable
self.config.set(self.SECTION, key, u'\n'.join([self._escape(x) for x in val] + [u';']))
else:
raise NotImplementedError()
def delete(self, key):
self.config.remove_option(self.SECTION, key)
def save(self):
with codecs.open(self.filename, 'w', 'utf-8') as h:
self.config.write(h)
def close(self):
self.save()
self.config = None
def _escape(self, val):
return str(val).replace(u'\\', u'\\\\').replace(u'\n', u'\\n').replace(u';', u'\\;')
def _unescape(self, val):
chars = list(val)
i = 0
while i < len(chars):
if chars[i] == '\\':
chars.pop(i)
if chars[i] == 'n':
chars[i] = '\n'
i += 1
return u''.join(chars)
else: # ???
def __init__(self):
raise NotImplementedError('Implement me')
# Common
def get_password(self, account):
try:
import keyring
return keyring.get_password(self.identifier, account)
except ImportError:
return None
def set_password(self, account, password):
try:
import keyring
keyring.set_password(self.identifier, account, password)
except ImportError:
pass
def delete_password(self, account):
try:
import keyring
keyring.delete_password(self.identifier, account)
except:
pass # don't care - silently fail
class MrxsFile(object):
def __init__(self, filename):
# Split filename
dirname, ext = os.path.splitext(filename)
if ext != ".mrxs":
raise UnrecognizedFile
# Parse slidedat
self._slidedatfile = os.path.join(dirname, "Slidedat.ini")
self._dat = RawConfigParser()
self._dat.optionxform = str
try:
with open(self._slidedatfile, "rb") as fh:
self._have_bom = fh.read(len(UTF8_BOM)) == UTF8_BOM
if not self._have_bom:
fh.seek(0)
self._dat.readfp(fh)
except IOError:
raise UnrecognizedFile
# Get file paths
self._indexfile = os.path.join(
dirname, self._dat.get(MRXS_HIERARCHICAL, "INDEXFILE"))
self._datafiles = [
os.path.join(dirname, self._dat.get("DATAFILE", "FILE_%d" % i))
for i in range(self._dat.getint("DATAFILE", "FILE_COUNT"))
]
# Build levels
self._make_levels()
def _make_levels(self):
self._levels = {}
self._level_list = []
layer_count = self._dat.getint(MRXS_HIERARCHICAL, "NONHIER_COUNT")
for layer_id in range(layer_count):
level_count = self._dat.getint(MRXS_HIERARCHICAL,
"NONHIER_%d_COUNT" % layer_id)
for level_id in range(level_count):
level = MrxsNonHierLevel(self._dat, layer_id, level_id,
len(self._level_list))
self._levels[(level.layer_name, level.name)] = level
self._level_list.append(level)
@classmethod
def _read_int32(cls, f):
buf = f.read(4)
if len(buf) != 4:
raise IOError("Short read")
return struct.unpack("<i", buf)[0]
@classmethod
def _assert_int32(cls, f, value):
v = cls._read_int32(f)
if v != value:
raise ValueError("%d != %d" % (v, value))
def _get_data_location(self, record):
with open(self._indexfile, "rb") as fh:
fh.seek(MRXS_NONHIER_ROOT_OFFSET)
# seek to record
table_base = self._read_int32(fh)
fh.seek(table_base + record * 4)
# seek to list head
list_head = self._read_int32(fh)
fh.seek(list_head)
# seek to data page
self._assert_int32(fh, 0)
page = self._read_int32(fh)
fh.seek(page)
# check pagesize
self._assert_int32(fh, 1)
# read rest of prologue
self._read_int32(fh)
self._assert_int32(fh, 0)
self._assert_int32(fh, 0)
# read values
position = self._read_int32(fh)
size = self._read_int32(fh)
fileno = self._read_int32(fh)
return (self._datafiles[fileno], position, size)
def _zero_record(self, record):
path, offset, length = self._get_data_location(record)
with open(path, "r+b") as fh:
fh.seek(0, 2)
do_truncate = fh.tell() == offset + length
if DEBUG:
if do_truncate:
print("Truncating", path, "to", offset)
else:
print("Zeroing", path, "at", offset, "for", length)
fh.seek(offset)
buf = fh.read(len(JPEG_SOI))
if buf != JPEG_SOI:
raise IOError("Unexpected data in nonhier image")
if do_truncate:
fh.truncate(offset)
else:
fh.seek(offset)
fh.write("\0" * length)
def _delete_index_record(self, record):
if DEBUG:
print("Deleting record", record)
with open(self._indexfile, "r+b") as fh:
entries_to_move = len(self._level_list) - record - 1
if entries_to_move == 0:
return
# get base of table
fh.seek(MRXS_NONHIER_ROOT_OFFSET)
table_base = self._read_int32(fh)
# read tail of table
fh.seek(table_base + (record + 1) * 4)
buf = fh.read(entries_to_move * 4)
if len(buf) != entries_to_move * 4:
raise IOError("Short read")
# overwrite the target record
fh.seek(table_base + record * 4)
fh.write(buf)
def _hier_keys_for_level(self, level):
ret = []
for k, _ in self._dat.items(MRXS_HIERARCHICAL):
if k == level.key_prefix or k.startswith(level.key_prefix + "_"):
ret.append(k)
return ret
def _rename_section(self, old, new):
if self._dat.has_section(old):
if DEBUG:
print("[%s] -> [%s]" % (old, new))
self._dat.add_section(new)
for k, v in self._dat.items(old):
self._dat.set(new, k, v)
self._dat.remove_section(old)
elif DEBUG:
print("[%s] does not exist" % old)
def _delete_section(self, section):
if DEBUG:
print("Deleting [%s]" % section)
self._dat.remove_section(section)
def _set_key(self, section, key, value):
if DEBUG:
prev = self._dat.get(section, key)
print("[%s] %s: %s -> %s" % (section, key, prev, value))
self._dat.set(section, key, value)
def _rename_key(self, section, old, new):
if DEBUG:
print("[%s] %s -> %s" % (section, old, new))
v = self._dat.get(section, old)
self._dat.remove_option(section, old)
self._dat.set(section, new, v)
def _delete_key(self, section, key):
if DEBUG:
print("Deleting [%s] %s" % (section, key))
self._dat.remove_option(section, key)
def _write(self):
buf = io.StringIO()
self._dat.write(buf)
with open(self._slidedatfile, "wb") as fh:
if self._have_bom:
fh.write(bytearray(UTF8_BOM))
fh.write(bytearray(buf.getvalue().replace("\n", "\r\n")))
def delete_level(self, layer_name, level_name):
level = self._levels[(layer_name, level_name)]
record = level.record
# Zero image data
self._zero_record(record)
# Delete pointer from nonhier table in index
self._delete_index_record(record)
# Remove slidedat keys
for k in self._hier_keys_for_level(level):
self._delete_key(MRXS_HIERARCHICAL, k)
# Remove slidedat section
self._delete_section(level.section)
# Rename section and keys for subsequent levels in the layer
prev_level = level
for cur_level in self._level_list[record + 1:]:
if cur_level.layer_id != prev_level.layer_id:
break
for k in self._hier_keys_for_level(cur_level):
new_k = k.replace(cur_level.key_prefix, prev_level.key_prefix,
1)
self._rename_key(MRXS_HIERARCHICAL, k, new_k)
self._set_key(MRXS_HIERARCHICAL, prev_level.section_key,
prev_level.section)
self._rename_section(cur_level.section, prev_level.section)
prev_level = cur_level
# Update level count within layer
count_k = "NONHIER_%d_COUNT" % level.layer_id
count_v = self._dat.getint(MRXS_HIERARCHICAL, count_k)
self._set_key(MRXS_HIERARCHICAL, count_k, count_v - 1)
# Write slidedat
self._write()
# Refresh metadata
self._make_levels()
def main(original_args=None):
positionals, args = split_args_in_optional_and_positional(
sys.argv[1:] if original_args is None else original_args)
parser1 = argparse.ArgumentParser(add_help=False)
parser1.add_argument('--config-file',
default='.bumpversion.cfg',
metavar='FILE',
help='Config file to read most of the variables from',
required=False)
parser1.add_argument('--verbose',
action='count',
default=0,
help='Print verbose logging to stderr',
required=False)
known_args, remaining_argv = parser1.parse_known_args(args)
if len(logger.handlers) == 0:
ch = logging.StreamHandler(sys.stderr)
logformatter = logging.Formatter('%(message)s')
ch.setFormatter(logformatter)
logger.addHandler(ch)
log_level = {
0: logging.WARNING,
1: logging.INFO,
2: logging.DEBUG,
}.get(known_args.verbose, logging.DEBUG)
logger.setLevel(log_level)
logger.debug("Starting {}".format(DESCRIPTION))
defaults = {}
vcs_info = {}
for vcs in VCS:
if vcs.is_usable():
vcs_info.update(vcs.latest_tag_info())
if 'current_version' in vcs_info:
defaults['current_version'] = vcs_info['current_version']
config = None
if os.path.exists(known_args.config_file):
config = RawConfigParser()
config.readfp(io.open(known_args.config_file, 'rt', encoding='utf-8'))
log_config = StringIO()
config.write(log_config)
logger.info("Reading config file {}:".format(known_args.config_file))
logger.info(log_config.getvalue())
defaults.update(dict(config.items("bumpversion")))
for listvaluename in ("serialize", ):
try:
value = config.get("bumpversion", listvaluename)
defaults[listvaluename] = list(
filter(None, (x.strip() for x in value.splitlines())))
except NoOptionError:
pass # no default value then ;)
for boolvaluename in ("commit", "tag", "dry_run"):
try:
defaults[boolvaluename] = config.getboolean(
"bumpversion", boolvaluename)
except NoOptionError:
pass # no default value then ;)
else:
message = "Could not read config file at {}".format(
known_args.config_file)
if known_args.config_file != parser1.get_default('config_file'):
raise argparse.ArgumentTypeError(message)
else:
logger.info(message)
parser2 = argparse.ArgumentParser(add_help=False, parents=[parser1])
parser2.set_defaults(**defaults)
parser2.add_argument('--current-version',
metavar='VERSION',
help='Version that needs to be updated',
required=False)
parser2.add_argument('--parse',
metavar='REGEX',
help='Regex parsing the version string',
default=defaults.get(
"parse",
'(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)'))
parser2.add_argument('--serialize',
metavar='FORMAT',
action=DiscardDefaultIfSpecifiedAppendAction,
help='How to format what is parsed back to a version',
default=defaults.get(
"serialize", [str('{major}.{minor}.{patch}')]))
known_args, remaining_argv = parser2.parse_known_args(args)
defaults.update(vars(known_args))
assert type(known_args.serialize) == list
time_context = {
'now': datetime.now(),
'utcnow': datetime.utcnow(),
}
try:
v = Version(
known_args.parse,
known_args.serialize,
context=dict(
list(time_context.items()) + list(prefixed_environ().items()) +
list(vcs_info.items())))
except sre_constants.error as e:
sys.exit(1)
if not 'new_version' in defaults and known_args.current_version:
v.parse(known_args.current_version)
if len(positionals) > 0:
v.bump(positionals[0])
try:
defaults['new_version'] = v.serialize()
except MissingValueForSerializationException as e:
logger.info("Opportunistic finding of new_version failed: " +
e.message)
except IncompleteVersionRepresenationException as e:
logger.info("Opportunistic finding of new_version failed: " +
e.message)
except KeyError as e:
logger.info("Opportunistic finding of new_version failed")
parser3 = argparse.ArgumentParser(
description=DESCRIPTION,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
conflict_handler='resolve',
parents=[parser2],
)
parser3.set_defaults(**defaults)
parser3.add_argument('--current-version',
metavar='VERSION',
help='Version that needs to be updated',
required=not 'current_version' in defaults)
parser3.add_argument('--dry-run',
'-n',
action='store_true',
default=False,
help="Don't write any files, just pretend.")
parser3.add_argument('--new-version',
metavar='VERSION',
help='New version that should be in the files',
required=not 'new_version' in defaults)
commitgroup = parser3.add_mutually_exclusive_group()
commitgroup.add_argument('--commit',
action='store_true',
dest="commit",
help='Commit to version control',
default=defaults.get("commit", False))
commitgroup.add_argument('--no-commit',
action='store_false',
dest="commit",
help='Do not commit to version control',
default=argparse.SUPPRESS)
taggroup = parser3.add_mutually_exclusive_group()
taggroup.add_argument('--tag',
action='store_true',
dest="tag",
default=defaults.get("tag", False),
help='Create a tag in version control')
taggroup.add_argument('--no-tag',
action='store_false',
dest="tag",
help='Do not create a tag in version control',
default=argparse.SUPPRESS)
parser3.add_argument('--tag-name',
metavar='TAG_NAME',
help='Tag name (only works with --tag)',
default=defaults.get('tag_name', 'v{new_version}'))
parser3.add_argument(
'--message',
'-m',
metavar='COMMIT_MSG',
help='Commit message',
default=defaults.get(
'message', 'Bump version: {current_version} → {new_version}'))
files = []
if 'files' in defaults:
assert defaults['files'] != None
files = defaults['files'].split(' ')
parser3.add_argument('part', help='Part of the version to be bumped.')
parser3.add_argument('files',
metavar='file',
nargs='*',
help='Files to change',
default=files)
args = parser3.parse_args(remaining_argv + positionals)
if args.dry_run:
logger.info("Dry run active, won't touch any files.")
logger.info("New version will be '{}'".format(args.new_version))
files = files or positionals[1:]
for vcs in VCS:
if vcs.is_usable():
vcs.assert_nondirty()
break
else:
vcs = None
# make sure files exist and contain version string
logger.info("Asserting files {} contain string '{}':".format(
", ".join(files), args.current_version))
for path in files:
with io.open(path, 'rb') as f:
found_before = False
for lineno, line in enumerate(f.readlines()):
if args.current_version in line.decode('utf-8'):
found_before = True
logger.info("Found '{}' in {} at line {}: {}".format(
args.current_version, path, lineno,
line.decode('utf-8').rstrip()))
assert found_before, 'Did not find string {} in file {}'.format(
args.current_version, path)
# change version string in files
for path in files:
with io.open(path, 'rb') as f:
before = f.read().decode('utf-8')
after = before.replace(args.current_version, args.new_version)
logger.info("{} file {}:".format(
"Would change" if args.dry_run else "Changing",
path,
))
logger.info("\n".join(
list(
unified_diff(before.splitlines(),
after.splitlines(),
lineterm="",
fromfile="a/" + path,
tofile="b/" + path))))
if not args.dry_run:
with io.open(path, 'wt', encoding='utf-8') as f:
f.write(after)
commit_files = files
if config:
config.remove_option('bumpversion', 'new_version')
config.set('bumpversion', 'current_version', args.new_version)
s = StringIO()
try:
config.write(s)
logger.info("{} to config file {}:".format(
"Would write" if args.dry_run else "Writing",
known_args.config_file,
))
logger.info(log_config.getvalue())
if not args.dry_run:
with io.open(known_args.config_file, 'wb') as f:
f.write(s.getvalue().encode('utf-8'))
except UnicodeEncodeError:
warnings.warn(
"Unable to write UTF-8 to config file, because of an old configparser version. "
"Update with `pip install --upgrade configparser`.")
commit_files.append(known_args.config_file)
if not vcs:
return
assert vcs.is_usable(), "Did find '{}' unusable, unable to commit.".format(
vcs.__name__)
do_commit = (not args.dry_run) and args.commit
do_tag = (not args.dry_run) and args.tag
logger.info("{} {} commit".format(
"Would prepare" if not do_commit else "Preparing",
vcs.__name__,
))
for path in commit_files:
logger.info("{} changes in file '{}' to {}".format(
"Would add" if not do_commit else "Adding",
path,
vcs.__name__,
))
if do_commit:
vcs.add_path(path)
vcs_context = {
"current_version": args.current_version,
"new_version": args.new_version,
}
vcs_context.update(time_context)
vcs_context.update(prefixed_environ())
commit_message = args.message.format(**vcs_context)
logger.info("{} to {} with message '{}'".format(
"Would commit" if not do_commit else "Committing",
vcs.__name__,
commit_message,
))
if do_commit:
vcs.commit(message=commit_message)
tag_name = args.tag_name.format(**vcs_context)
logger.info("{} '{}' in {}".format(
"Would tag" if not do_tag else "Tagging", tag_name, vcs.__name__))
if do_tag:
vcs.tag(tag_name)
class MrxsFile(object):
def __init__(self, filename):
# Split filename
dirname, ext = os.path.splitext(filename)
if ext != '.mrxs':
raise UnrecognizedFile
# Parse slidedat
self._slidedatfile = os.path.join(dirname, 'Slidedat.ini')
self._slidedatfile_upperdir = os.path.join(dirname, '../Slidedat.ini')
self._dat = RawConfigParser()
self._dat.optionxform = str
self._anonymize_meta(dirname)
try:
with open(self._slidedatfile, 'r', encoding="utf-8-sig") as fh:
self._have_bom = (fh.read(len(UTF8_BOM)) == UTF8_BOM)
if not self._have_bom:
fh.seek(0)
self._dat.read_file(fh)
except IOError:
raise UnrecognizedFile
# Get file paths
self._indexfile = os.path.join(
dirname, self._dat.get(MRXS_HIERARCHICAL, 'INDEXFILE'))
self._datafiles = [
os.path.join(dirname, self._dat.get('DATAFILE', 'FILE_%d' % i))
for i in range(self._dat.getint('DATAFILE', 'FILE_COUNT'))
]
# Build levels
self._make_levels()
def _anonymize_meta(self, dirname):
for filename in [
os.path.join(dirname, 'Slidedat.ini'),
os.path.join(dirname, '../Slidedat.ini')
]:
filedata = ''
if os.path.exists(filename):
with open(filename, 'r') as fh:
filedata = fh.read()
filedata = re.sub(r"SLIDE_NAME.*", "SLIDE_NAME = None",
filedata)
filedata = re.sub(r"PROJECT_NAME.*", "PROJECT_NAME = None",
filedata)
filedata = re.sub(r"SLIDE_CREATIONDATETIME.*",
"SLIDE_CREATIONDATETIME = None",
filedata)
fh.close
with open(filename, 'w') as fh:
fh.write(filedata)
fh.close()
def _make_levels(self):
self._levels = {}
self._level_list = []
layer_count = self._dat.getint(MRXS_HIERARCHICAL, 'NONHIER_COUNT')
for layer_id in range(layer_count):
level_count = self._dat.getint(MRXS_HIERARCHICAL,
'NONHIER_%d_COUNT' % layer_id)
for level_id in range(level_count):
level = MrxsNonHierLevel(self._dat, layer_id, level_id,
len(self._level_list))
self._levels[(level.layer_name, level.name)] = level
self._level_list.append(level)
@classmethod
def _read_int32(cls, f):
buf = f.read(4)
if len(buf) != 4:
raise IOError('Short read')
return struct.unpack('<i', buf)[0]
@classmethod
def _assert_int32(cls, f, value):
v = cls._read_int32(f)
if v != value:
raise ValueError('%d != %d' % (v, value))
def _get_data_location(self, record):
with open(self._indexfile, 'rb') as fh:
fh.seek(MRXS_NONHIER_ROOT_OFFSET)
# seek to record
table_base = self._read_int32(fh)
fh.seek(table_base + record * 4)
# seek to list head
list_head = self._read_int32(fh)
fh.seek(list_head)
# seek to data page
self._assert_int32(fh, 0)
page = self._read_int32(fh)
fh.seek(page)
# check pagesize
self._assert_int32(fh, 1)
# read rest of prologue
self._read_int32(fh)
self._assert_int32(fh, 0)
self._assert_int32(fh, 0)
# read values
position = self._read_int32(fh)
size = self._read_int32(fh)
fileno = self._read_int32(fh)
return (self._datafiles[fileno], position, size)
def _zero_record(self, record):
path, offset, length = self._get_data_location(record)
with open(path, 'r+b') as fh:
fh.seek(0, 2)
do_truncate = (fh.tell() == offset + length)
if DEBUG:
if do_truncate:
print('Truncating', path, 'to', offset)
else:
print('Zeroing', path, 'at', offset, 'for', length)
fh.seek(offset)
buf = fh.read(len(JPEG_SOI))
# print(buf)
# exit()
if buf != JPEG_SOI:
raise IOError('Unexpected data in nonhier image')
if do_truncate:
fh.truncate(offset)
else:
fh.seek(offset)
fh.write('\0' * length)
def _delete_index_record(self, record):
if DEBUG:
print('Deleting record', record)
with open(self._indexfile, 'r+b') as fh:
entries_to_move = len(self._level_list) - record - 1
if entries_to_move == 0:
return
# get base of table
fh.seek(MRXS_NONHIER_ROOT_OFFSET)
table_base = self._read_int32(fh)
# read tail of table
fh.seek(table_base + (record + 1) * 4)
buf = fh.read(entries_to_move * 4)
if len(buf) != entries_to_move * 4:
raise IOError('Short read')
# overwrite the target record
fh.seek(table_base + record * 4)
fh.write(buf)
def _hier_keys_for_level(self, level):
ret = []
for k, _ in self._dat.items(MRXS_HIERARCHICAL):
if k == level.key_prefix or k.startswith(level.key_prefix + '_'):
ret.append(k)
return ret
def _rename_section(self, old, new):
if self._dat.has_section(old):
if DEBUG:
print('[%s] -> [%s]' % (old, new))
self._dat.add_section(new)
for k, v in self._dat.items(old):
self._dat.set(new, k, v)
self._dat.remove_section(old)
elif DEBUG:
print('[%s] does not exist' % old)
def _delete_section(self, section):
if DEBUG:
print('Deleting [%s]' % section)
self._dat.remove_section(section)
def _set_key(self, section, key, value):
if DEBUG:
prev = self._dat.get(section, key)
print('[%s] %s: %s -> %s' % (section, key, prev, value))
self._dat.set(section, key, value)
def _rename_key(self, section, old, new):
if DEBUG:
print('[%s] %s -> %s' % (section, old, new))
v = self._dat.get(section, old)
self._dat.remove_option(section, old)
self._dat.set(section, new, v)
def _delete_key(self, section, key):
if DEBUG:
print('Deleting [%s] %s' % (section, key))
self._dat.remove_option(section, key)
def _write(self):
buf = StringIO()
self._dat.write(buf)
with open(self._slidedatfile, 'wb') as fh:
if self._have_bom:
fh.write(UTF8_BOM.encode())
fh.write(buf.getvalue().replace('\n', '\r\n').encode())
def delete_level(self, layer_name, level_name):
level = self._levels[(layer_name, level_name)]
record = level.record
# Zero image data
self._zero_record(record)
# Delete pointer from nonhier table in index
self._delete_index_record(record)
# Remove slidedat keys
for k in self._hier_keys_for_level(level):
self._delete_key(MRXS_HIERARCHICAL, k)
# Remove slidedat section
self._delete_section(level.section)
# Rename section and keys for subsequent levels in the layer
prev_level = level
for cur_level in self._level_list[record + 1:]:
if cur_level.layer_id != prev_level.layer_id:
break
for k in self._hier_keys_for_level(cur_level):
new_k = k.replace(cur_level.key_prefix, prev_level.key_prefix,
1)
self._rename_key(MRXS_HIERARCHICAL, k, new_k)
self._set_key(MRXS_HIERARCHICAL, prev_level.section_key,
prev_level.section)
self._rename_section(cur_level.section, prev_level.section)
prev_level = cur_level
# Update level count within layer
count_k = 'NONHIER_%d_COUNT' % level.layer_id
count_v = self._dat.getint(MRXS_HIERARCHICAL, count_k)
self._set_key(MRXS_HIERARCHICAL, count_k, count_v - 1)
# Write slidedat
self._write()
# Refresh metadata
self._make_levels()
class BaseConfigStore(object):
# the actual config store functionality
def __init__(self, name, *arg, **kw):
super(BaseConfigStore, self).__init__(*arg, **kw)
self.dirty = False
self.config = RawConfigParser()
config_dir = os.environ.get('PHOTINI_CONFIG')
if config_dir:
config_dir = os.path.expanduser(config_dir)
elif hasattr(appdirs, 'user_config_dir'):
config_dir = appdirs.user_config_dir('photini')
else:
config_dir = appdirs.user_data_dir('photini')
if not os.path.isdir(config_dir):
os.makedirs(config_dir, mode=stat.S_IRWXU)
self.file_name = os.path.join(config_dir, name + '.ini')
if os.path.isfile(self.file_name):
kwds = {'encoding': 'utf-8'}
with open(self.file_name, 'r', **kwds) as fp:
self.config.read_file(fp)
self.has_section = self.config.has_section
def get(self, section, option, default=None):
if self.config.has_option(section, option):
return self.config.get(section, option)
if default is not None:
self.set(section, option, default)
return default
def set(self, section, option, value):
if not self.config.has_section(section):
self.config.add_section(section)
elif (self.config.has_option(section, option)
and self.config.get(section, option) == value):
return
self.config.set(section, option, value)
self.dirty = True
def delete(self, section, option):
if not self.config.has_section(section):
return
if self.config.has_option(section, option):
self.config.remove_option(section, option)
if not self.config.options(section):
self.config.remove_section(section)
self.dirty = True
def remove_section(self, section):
if not self.config.has_section(section):
return
for option in self.config.options(section):
self.config.remove_option(section, option)
self.config.remove_section(section)
self.dirty = True
def save(self):
if not self.dirty:
return
kwds = {'encoding': 'utf-8'}
with open(self.file_name, 'w', **kwds) as fp:
self.config.write(fp)
os.chmod(self.file_name, stat.S_IRUSR | stat.S_IWUSR)
self.dirty = False
def remove_option(self, section, option):
if self.has_option(section, option):
RawConfigParser.remove_option(self, section, option)
class Config(object):
"""A wrapper around RawConfigParser.
Provides a ``defaults`` attribute of the same type which can be used
to set default values.
"""
def __init__(self, version=None, _defaults=True):
"""Use read() to read in an existing config file.
version should be an int starting with 0 that gets incremented if you
want to register a new upgrade function. If None, upgrade is disabled.
"""
self._config = ConfigParser(dict_type=_sorted_dict)
self.defaults = None
if _defaults:
self.defaults = Config(_defaults=False)
self._version = version
self._loaded_version = None
self._upgrade_funcs = []
def _do_upgrade(self, func):
assert self._loaded_version is not None
assert self._version is not None
old_version = self._loaded_version
new_version = self._version
if old_version != new_version:
print_d("Config upgrade: %d->%d (%r)" % (
old_version, new_version, func))
func(self, old_version, new_version)
def get_version(self):
"""Get the version of the loaded config file (for testing only)
Raises Error if no file was loaded or versioning is disabled.
"""
if self._version is None:
raise Error("Versioning disabled")
if self._loaded_version is None:
raise Error("No file loaded")
return self._loaded_version
def register_upgrade_function(self, function):
"""Register an upgrade function that gets called at each read()
if the current config version and the loaded version don't match.
Can also be registered after read was called.
function(config, old_version: int, new_version: int) -> None
"""
if self._version is None:
raise Error("Versioning disabled")
self._upgrade_funcs.append(function)
# after read(), so upgrade now
if self._loaded_version is not None:
self._do_upgrade(function)
return function
def reset(self, section, option):
"""Reset the value to the default state"""
assert self.defaults is not None
try:
self._config.remove_option(section, option)
except NoSectionError:
pass
def options(self, section):
"""Returns a list of options available in the specified section."""
try:
options = self._config.options(section)
except NoSectionError:
if self.defaults:
return self.defaults.options(section)
raise
else:
if self.defaults:
try:
options.extend(self.defaults.options(section))
options = list_unique(options)
except NoSectionError:
pass
return options
def get(self, section, option, default=_DEFAULT):
"""get(section, option[, default]) -> str
If default is not given or set, raises Error in case of an error
"""
try:
return self._config.get(section, option)
except Error:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.get(section, option)
except Error:
pass
raise
return default
def gettext(self, *args, **kwargs):
value = self.get(*args, **kwargs)
# make sure there are no surrogates
value.encode("utf-8")
return value
def getbytes(self, section, option, default=_DEFAULT):
try:
value = self._config.get(section, option)
value = value.encode("utf-8", "surrogateescape")
return value
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getbytes(section, option)
except Error:
pass
raise Error(e)
return default
def getboolean(self, section, option, default=_DEFAULT):
"""getboolean(section, option[, default]) -> bool
If default is not given or set, raises Error in case of an error
"""
try:
return self._config.getboolean(section, option)
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getboolean(section, option)
except Error:
pass
raise Error(e)
return default
def getint(self, section, option, default=_DEFAULT):
"""getint(section, option[, default]) -> int
If default is not give or set, raises Error in case of an error
"""
try:
return int(self._config.getfloat(section, option))
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getint(section, option)
except Error:
pass
raise Error(e)
return default
def getfloat(self, section, option, default=_DEFAULT):
"""getfloat(section, option[, default]) -> float
If default is not give or set, raises Error in case of an error
"""
try:
return self._config.getfloat(section, option)
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getfloat(section, option)
except Error:
pass
raise Error(e)
return default
def getstringlist(self, section, option, default=_DEFAULT):
"""getstringlist(section, option[, default]) -> list
If default is not given or set, raises Error in case of an error.
Gets a list of strings, using CSV to parse and delimit.
"""
try:
value = self._config.get(section, option)
parser = csv.reader(
[value], lineterminator='\n', quoting=csv.QUOTE_MINIMAL)
try:
vals = next(parser)
except (csv.Error, ValueError) as e:
raise Error(e)
return vals
except Error as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getstringlist(section, option)
except Error:
pass
raise Error(e)
return default
def setstringlist(self, section, option, values):
"""Saves a list of unicode strings using the csv module"""
sw = StringIO()
values = [str(v) for v in values]
writer = csv.writer(sw, lineterminator='\n', quoting=csv.QUOTE_MINIMAL)
writer.writerow(values)
self.set(section, option, sw.getvalue())
def setlist(self, section, option, values, sep=","):
"""Saves a list of str using ',' as a separator and \\ for escaping"""
values = [str(v) for v in values]
joined = join_escape(values, sep)
self.set(section, option, joined)
def getlist(self, section, option, default=_DEFAULT, sep=","):
"""Returns a str list saved with setlist()"""
try:
value = self._config.get(section, option)
return split_escape(value, sep)
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getlist(section, option, sep=sep)
except Error:
pass
raise Error(e)
return default
def set(self, section, option, value):
"""Saves the string representation for the passed value
Don't pass unicode, encode first.
"""
if isinstance(value, bytes):
raise TypeError("use setbytes")
# RawConfigParser only allows string values but doesn't
# scream if they are not (and it only fails before the
# first config save..)
if not isinstance(value, str):
value = str(value)
try:
self._config.set(section, option, value)
except NoSectionError:
if self.defaults and self.defaults.has_section(section):
self._config.add_section(section)
self._config.set(section, option, value)
else:
raise
def settext(self, section, option, value):
value = str(value)
# make sure there are no surrogates
value.encode("utf-8")
self.set(section, option, value)
def setbytes(self, section, option, value):
assert isinstance(value, bytes)
value = value.decode("utf-8", "surrogateescape")
self.set(section, option, value)
def write(self, filename):
"""Write config to filename.
Can raise EnvironmentError
"""
assert isinstance(filename, fsnative)
mkdir(os.path.dirname(filename))
# temporary set the new version for saving
if self._version is not None:
self.add_section("__config__")
self.set("__config__", "version", self._version)
try:
with atomic_save(filename, "wb") as fileobj:
temp = StringIO()
self._config.write(temp)
data = temp.getvalue().encode("utf-8", "surrogateescape")
fileobj.write(data)
finally:
if self._loaded_version is not None:
self.set("__config__", "version", self._loaded_version)
def clear(self):
"""Remove all sections."""
for section in self._config.sections():
self._config.remove_section(section)
def is_empty(self):
"""Whether the config has any sections"""
return not self._config.sections()
def read(self, filename):
"""Reads the config from `filename` if the file exists,
otherwise does nothing
Can raise EnvironmentError, Error.
"""
try:
with open(filename, "rb") as fileobj:
fileobj = StringIO(
fileobj.read().decode("utf-8", "surrogateescape"))
self._config.readfp(fileobj, filename)
except (IOError, OSError):
return
# don't upgrade if we just created a new config
if self._version is not None:
self._loaded_version = self.getint("__config__", "version", -1)
for func in self._upgrade_funcs:
self._do_upgrade(func)
def has_option(self, section, option):
"""If the given section exists, and contains the given option"""
return self._config.has_option(section, option) or (
self.defaults and self.defaults.has_option(section, option))
def has_section(self, section):
"""If the given section exists"""
return self._config.has_section(section) or (
self.defaults and self.defaults.has_section(section))
def remove_option(self, section, option):
"""Remove the specified option from the specified section
Can raise Error.
"""
return self._config.remove_option(section, option)
def add_section(self, section):
"""Add a section named section to the instance if it not already
exists."""
if not self._config.has_section(section):
self._config.add_section(section)
def main(original_args=None):
positionals, args = split_args_in_optional_and_positional(
sys.argv[1:] if original_args is None else original_args
)
if len(positionals[1:]) > 2:
warnings.warn("Giving multiple files on the command line will be deprecated, please use [bumpversion:file:...] in a config file.", PendingDeprecationWarning)
parser1 = argparse.ArgumentParser(add_help=False)
parser1.add_argument(
'--config-file', metavar='FILE',
default=argparse.SUPPRESS, required=False,
help='Config file to read most of the variables from (default: .bumpversion.cfg)')
parser1.add_argument(
'--verbose', action='count', default=0,
help='Print verbose logging to stderr', required=False)
parser1.add_argument(
'--list', action='store_true', default=False,
help='List machine readable information', required=False)
parser1.add_argument(
'--allow-dirty', action='store_true', default=False,
help="Don't abort if working directory is dirty", required=False)
known_args, remaining_argv = parser1.parse_known_args(args)
logformatter = logging.Formatter('%(message)s')
if len(logger.handlers) == 0:
ch = logging.StreamHandler(sys.stderr)
ch.setFormatter(logformatter)
logger.addHandler(ch)
if len(logger_list.handlers) == 0:
ch2 = logging.StreamHandler(sys.stdout)
ch2.setFormatter(logformatter)
logger_list.addHandler(ch2)
if known_args.list:
logger_list.setLevel(1)
log_level = {
0: logging.WARNING,
1: logging.INFO,
2: logging.DEBUG,
}.get(known_args.verbose, logging.DEBUG)
logger.setLevel(log_level)
logger.debug("Starting {}".format(DESCRIPTION))
defaults = {}
vcs_info = {}
for vcs in VCS:
if vcs.is_usable():
vcs_info.update(vcs.latest_tag_info())
if 'current_version' in vcs_info:
defaults['current_version'] = vcs_info['current_version']
config = RawConfigParser('')
# don't transform keys to lowercase (which would be the default)
config.optionxform = lambda option: option
config.add_section('bumpversion')
explicit_config = hasattr(known_args, 'config_file')
if explicit_config:
config_file = known_args.config_file
elif not os.path.exists('.bumpversion.cfg') and \
os.path.exists('setup.cfg'):
config_file = 'setup.cfg'
else:
config_file = '.bumpversion.cfg'
config_file_exists = os.path.exists(config_file)
part_configs = {}
files = []
if config_file_exists:
logger.info("Reading config file {}:".format(config_file))
logger.info(io.open(config_file, 'rt', encoding='utf-8').read())
config.readfp(io.open(config_file, 'rt', encoding='utf-8'))
log_config = StringIO()
config.write(log_config)
if 'files' in dict(config.items("bumpversion")):
warnings.warn(
"'files =' configuration is will be deprecated, please use [bumpversion:file:...]",
PendingDeprecationWarning
)
defaults.update(dict(config.items("bumpversion")))
for listvaluename in ("serialize",):
try:
value = config.get("bumpversion", listvaluename)
defaults[listvaluename] = list(filter(None, (x.strip() for x in value.splitlines())))
except NoOptionError:
pass # no default value then ;)
for boolvaluename in ("commit", "tag", "dry_run"):
try:
defaults[boolvaluename] = config.getboolean(
"bumpversion", boolvaluename)
except NoOptionError:
pass # no default value then ;)
for section_name in config.sections():
section_name_match = re.compile("^bumpversion:(file|part):(.+)").match(section_name)
if not section_name_match:
continue
section_prefix, section_value = section_name_match.groups()
section_config = dict(config.items(section_name))
if section_prefix == "part":
ThisVersionPartConfiguration = NumericVersionPartConfiguration
if 'values' in section_config:
section_config['values'] = list(filter(None, (x.strip() for x in section_config['values'].splitlines())))
ThisVersionPartConfiguration = ConfiguredVersionPartConfiguration
part_configs[section_value] = ThisVersionPartConfiguration(**section_config)
elif section_prefix == "file":
filename = section_value
if 'serialize' in section_config:
section_config['serialize'] = list(filter(None, (x.strip() for x in section_config['serialize'].splitlines())))
section_config['part_configs'] = part_configs
if not 'parse' in section_config:
section_config['parse'] = defaults.get("parse", '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)')
if not 'serialize' in section_config:
section_config['serialize'] = defaults.get('serialize', [str('{major}.{minor}.{patch}')])
if not 'search' in section_config:
section_config['search'] = defaults.get("search", '{current_version}')
if not 'replace' in section_config:
section_config['replace'] = defaults.get("replace", '{new_version}')
files.append(ConfiguredFile(filename, VersionConfig(**section_config)))
else:
message = "Could not read config file at {}".format(config_file)
if explicit_config:
raise argparse.ArgumentTypeError(message)
else:
logger.info(message)
parser2 = argparse.ArgumentParser(prog='bumpversion', add_help=False, parents=[parser1])
parser2.set_defaults(**defaults)
parser2.add_argument('--current-version', metavar='VERSION',
help='Version that needs to be updated', required=False)
parser2.add_argument('--parse', metavar='REGEX',
help='Regex parsing the version string',
default=defaults.get("parse", '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)'))
parser2.add_argument('--serialize', metavar='FORMAT',
action=DiscardDefaultIfSpecifiedAppendAction,
help='How to format what is parsed back to a version',
default=defaults.get("serialize", [str('{major}.{minor}.{patch}')]))
parser2.add_argument('--search', metavar='SEARCH',
help='Template for complete string to search',
default=defaults.get("search", '{current_version}'))
parser2.add_argument('--replace', metavar='REPLACE',
help='Template for complete string to replace',
default=defaults.get("replace", '{new_version}'))
known_args, remaining_argv = parser2.parse_known_args(args)
defaults.update(vars(known_args))
assert type(known_args.serialize) == list
context = dict(list(time_context.items()) + list(prefixed_environ().items()) + list(vcs_info.items()))
try:
vc = VersionConfig(
parse=known_args.parse,
serialize=known_args.serialize,
search=known_args.search,
replace=known_args.replace,
part_configs=part_configs,
)
except sre_constants.error as e:
sys.exit(1)
current_version = vc.parse(known_args.current_version) if known_args.current_version else None
new_version = None
if not 'new_version' in defaults and known_args.current_version:
try:
if current_version and len(positionals) > 0:
logger.info("Attempting to increment part '{}'".format(positionals[0]))
new_version = current_version.bump(positionals[0], vc.order())
logger.info("Values are now: " + keyvaluestring(new_version._values))
defaults['new_version'] = vc.serialize(new_version, context)
except MissingValueForSerializationException as e:
logger.info("Opportunistic finding of new_version failed: " + e.message)
except IncompleteVersionRepresenationException as e:
logger.info("Opportunistic finding of new_version failed: " + e.message)
except KeyError as e:
logger.info("Opportunistic finding of new_version failed")
parser3 = argparse.ArgumentParser(
prog='bumpversion',
description=DESCRIPTION,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
conflict_handler='resolve',
parents=[parser2],
)
parser3.set_defaults(**defaults)
parser3.add_argument('--current-version', metavar='VERSION',
help='Version that needs to be updated',
required=not 'current_version' in defaults)
parser3.add_argument('--dry-run', '-n', action='store_true',
default=False, help="Don't write any files, just pretend.")
parser3.add_argument('--new-version', metavar='VERSION',
help='New version that should be in the files',
required=not 'new_version' in defaults)
commitgroup = parser3.add_mutually_exclusive_group()
commitgroup.add_argument('--commit', action='store_true', dest="commit",
help='Commit to version control', default=defaults.get("commit", False))
commitgroup.add_argument('--no-commit', action='store_false', dest="commit",
help='Do not commit to version control', default=argparse.SUPPRESS)
taggroup = parser3.add_mutually_exclusive_group()
taggroup.add_argument('--tag', action='store_true', dest="tag", default=defaults.get("tag", False),
help='Create a tag in version control')
taggroup.add_argument('--no-tag', action='store_false', dest="tag",
help='Do not create a tag in version control', default=argparse.SUPPRESS)
parser3.add_argument('--tag-name', metavar='TAG_NAME',
help='Tag name (only works with --tag)',
default=defaults.get('tag_name', 'v{new_version}'))
parser3.add_argument('--message', '-m', metavar='COMMIT_MSG',
help='Commit message',
default=defaults.get('message', 'Bump version: {current_version} → {new_version}'))
file_names = []
if 'files' in defaults:
assert defaults['files'] != None
file_names = defaults['files'].split(' ')
parser3.add_argument('part',
help='Part of the version to be bumped.')
parser3.add_argument('files', metavar='file',
nargs='*',
help='Files to change', default=file_names)
args = parser3.parse_args(remaining_argv + positionals)
if args.dry_run:
logger.info("Dry run active, won't touch any files.")
if args.new_version:
new_version = vc.parse(args.new_version)
logger.info("New version will be '{}'".format(args.new_version))
file_names = file_names or positionals[1:]
for file_name in file_names:
files.append(ConfiguredFile(file_name, vc))
for vcs in VCS:
if vcs.is_usable():
try:
vcs.assert_nondirty()
except WorkingDirectoryIsDirtyException as e:
if not defaults['allow_dirty']:
logger.warn(
"{}\n\nUse --allow-dirty to override this if you know what you're doing.".format(e.message))
raise
break
else:
vcs = None
# make sure files exist and contain version string
logger.info("Asserting files {} contain the version string:".format(", ".join([str(f) for f in files])))
for f in files:
f.should_contain_version(current_version, context)
# change version string in files
for f in files:
f.replace(current_version, new_version, context, args.dry_run)
commit_files = [f.path for f in files]
config.set('bumpversion', 'new_version', args.new_version)
for key, value in config.items('bumpversion'):
logger_list.info("{}={}".format(key, value))
config.remove_option('bumpversion', 'new_version')
config.set('bumpversion', 'current_version', args.new_version)
new_config = StringIO()
try:
write_to_config_file = (not args.dry_run) and config_file_exists
logger.info("{} to config file {}:".format(
"Would write" if not write_to_config_file else "Writing",
config_file,
))
config.write(new_config)
logger.info(new_config.getvalue())
if write_to_config_file:
with io.open(config_file, 'wb') as f:
f.write(new_config.getvalue().encode('utf-8'))
except UnicodeEncodeError:
warnings.warn(
"Unable to write UTF-8 to config file, because of an old configparser version. "
"Update with `pip install --upgrade configparser`."
)
if config_file_exists:
commit_files.append(config_file)
if not vcs:
return
assert vcs.is_usable(), "Did find '{}' unusable, unable to commit.".format(vcs.__name__)
do_commit = (not args.dry_run) and args.commit
do_tag = (not args.dry_run) and args.tag
logger.info("{} {} commit".format(
"Would prepare" if not do_commit else "Preparing",
vcs.__name__,
))
for path in commit_files:
logger.info("{} changes in file '{}' to {}".format(
"Would add" if not do_commit else "Adding",
path,
vcs.__name__,
))
if do_commit:
vcs.add_path(path)
vcs_context = {
"current_version": args.current_version,
"new_version": args.new_version,
}
vcs_context.update(time_context)
vcs_context.update(prefixed_environ())
commit_message = args.message.format(**vcs_context)
logger.info("{} to {} with message '{}'".format(
"Would commit" if not do_commit else "Committing",
vcs.__name__,
commit_message,
))
if do_commit:
vcs.commit(message=commit_message)
tag_name = args.tag_name.format(**vcs_context)
logger.info("{} '{}' in {}".format(
"Would tag" if not do_tag else "Tagging",
tag_name,
vcs.__name__
))
if do_tag:
vcs.tag(tag_name)
class ConfigEditor:
def __init__(self):
self.config_parser = RawConfigParser()
self.config_file = None
def open(self, config_file):
"""
Open and read a config file
:param config_file: config file path
"""
logger.debug('Opening {}'.format(config_file))
self.config_file = config_file
if os.access(config_file, os.R_OK):
logger.debug('Parsing config file')
self.config_parser.read(config_file)
def read(self, section, option, fallback=None):
"""
Returns a config option value from config file
:param section: section where the option is stored
:param option: option name
:param fallback: (optional) fallback value
:return: a config option value
:rtype: string
"""
if self.config_file == None:
raise InvalidOperation('read')
if fallback is None:
return self.config_parser.get(section, option)
else:
return self.config_parser.get(section, option, fallback=fallback)
def readboolean(self, section, option, fallback=False):
"""
Returns a boolean config option value from config file
:param section: section where the option is stored
:param option: option name
:param fallback: (optional) fallback value
:return: a config option value
:rtype: boolean
"""
if self.config_file == None:
raise InvalidOperation('readboolean')
return self.config_parser.getboolean(section,
option,
fallback=fallback)
def write(self, section, option, value):
"""
Write a config option value in config object
:param section: section where the option is stored
:param option: option name
:param value: option value
"""
if self.config_file == None:
raise InvalidOperation('write')
if section != 'DEFAULT' and not self.config_parser.has_section(
section):
logger.debug('Adding new section {}'.format(section))
self.config_parser.add_section(section)
logger.debug('Adding {}.{} with value {}'.format(
section, option, value))
self.config_parser.set(section, option, value)
def remove(self, section, option):
"""
Remove a config option in config object
:param section: section where the option is stored
:param option: option name
:return: True if option is removed, False if not exist
:rtype: boolean
"""
if self.config_file == None:
raise InvalidOperation('remove')
logger.debug('Removing {}.{}'.format(section, option))
option_removed = self.config_parser.remove_option(section, option)
if section != 'DEFAULT' and option_removed:
if self.config_parser.items(section) == self.config_parser.items(
'DEFAULT'):
logger.debug('Removing empty section {}'.format(section))
self.config_parser.remove_section(section)
return option_removed
def remove_project(self, project):
"""
Remove a project (config section in config object)
:param project: section name
:return: True if section is removed, False if not exist
:rtype: boolean
"""
if self.config_file == None:
raise InvalidOperation('remove')
logger.debug('Removing {}'.format(project))
return self.config_parser.remove_section(project)
def list(self):
"""
List config sections
:return: list of projects (sections in config)
:rtype: list
"""
if self.config_file == None:
raise InvalidOperation('list')
return self.config_parser.sections()
def list_enabled_projects(self):
"""
Get the list of enabled projects
:return: list of enabled projects
:rtype: list
"""
if self.config_file == None:
raise InvalidOperation('list_enabled_projects')
try:
return self.config_parser.get('DEFAULT', 'sync_projects').split()
except NoOptionError:
return []
def enable_project(self, project):
"""
Enable a project adding it to sync_projects
:param project: project name
"""
if self.config_file == None:
raise InvalidOperation('enable_project')
logger.debug('Enabling project {}'.format(project))
enabled_projects = self.list_enabled_projects()
enabled_projects.append(project)
enabled_projects.sort()
self.config_parser.set('DEFAULT', 'sync_projects',
' '.join(enabled_projects))
def disable_project(self, project):
"""
Disable a project removing it from sync_projects
:param project: project name
:return: True if project is disabled, False if not
:rtype: boolean
"""
if self.config_file == None:
raise InvalidOperation('disable_project')
logger.debug('Disabling project {}'.format(project))
enabled_projects = self.list_enabled_projects()
try:
enabled_projects.remove(project)
self.config_parser.set('DEFAULT', 'sync_projects',
' '.join(enabled_projects))
return True
except ValueError:
logger.debug(
'Nothing to do, {} is not in enabled projects'.format(project))
return False
def has_project(self, project):
"""
Check if a project (a section in config) is present
:param project: section name
:return: True if section exists, False if not
:rtype: boolean
"""
if self.config_file == None:
raise InvalidOperation('has_project')
return self.config_parser.has_section(project)
def has_project_enabled(self, project):
"""
Check if a project is enabled
:param project: project name
:return: True if project is enabled, False if not
:rtype: boolean
"""
if self.config_file == None:
raise InvalidOperation('has_project_enabled')
return True if project in self.list_enabled_projects() else False
def save(self):
"""
Save the config object in config file
"""
if self.config_file == None:
raise InvalidOperation('save')
logger.debug('Saving config in config file')
with open(self.config_file, 'w') as configfile:
self.config_parser.write(configfile)
self.config_file = None
def clean(self):
"""
Cleans the config editor
"""
logger.debug('Cleaning config editor')
self.config_parser = RawConfigParser()
self.config_file = None
class ParamStore(object):
def __init__(self, root_dir, file_name):
self._lock = threading.Lock()
with self._lock:
if not os.path.isdir(root_dir):
raise RuntimeError('Directory "' + root_dir +
'" does not exist.')
self._path = os.path.join(root_dir, file_name)
self._dirty = False
# open config file
self._config = RawConfigParser()
self._config.read(self._path)
def flush(self):
if not self._dirty:
return
with self._lock:
self._dirty = False
with open(self._path, 'w') as of:
self._config.write(of)
def get(self, section, option, default=None):
"""Get a parameter value and return a string.
If default is specified and section or option are not defined
in the file, they are created and set to default, which is
then the return value.
"""
with self._lock:
if not self._config.has_option(section, option):
if default is not None:
self._set(section, option, default)
return default
return self._config.get(section, option)
def get_datetime(self, section, option, default=None):
result = self.get(section, option, default)
if result:
return WSDateTime.from_csv(result)
return result
def set(self, section, option, value):
"""Set option in section to string value."""
with self._lock:
self._set(section, option, value)
def _set(self, section, option, value):
if not self._config.has_section(section):
self._config.add_section(section)
elif (self._config.has_option(section, option)
and self._config.get(section, option) == value):
return
self._config.set(section, option, value)
self._dirty = True
def unset(self, section, option):
"""Remove option from section."""
with self._lock:
if not self._config.has_section(section):
return
if self._config.has_option(section, option):
self._config.remove_option(section, option)
self._dirty = True
if not self._config.options(section):
self._config.remove_section(section)
self._dirty = True
def main(original_args=None):
positionals, args = split_args_in_optional_and_positional(original_args)
parser1 = argparse.ArgumentParser(add_help=False)
parser1.add_argument(
"--config-file",
default=".bumpversion.cfg",
metavar="FILE",
help="Config file to read most of the variables from",
required=False,
)
known_args, remaining_argv = parser1.parse_known_args(args)
defaults = {}
vcs_info = {}
for vcs in VCS:
if vcs.is_usable():
vcs_info.update(vcs.latest_tag_info())
if "current_version" in vcs_info:
defaults["current_version"] = vcs_info["current_version"]
config = None
if os.path.exists(known_args.config_file):
config = RawConfigParser()
config.readfp(io.open(known_args.config_file, "rt", encoding="utf-8"))
defaults.update(dict(config.items("bumpversion")))
for boolvaluename in ("commit", "tag", "dry_run"):
try:
defaults[boolvaluename] = config.getboolean("bumpversion", boolvaluename)
except NoOptionError:
pass # no default value then ;)
elif known_args.config_file != parser1.get_default("config_file"):
raise argparse.ArgumentTypeError("Could not read config file at {}".format(known_args.config_file))
parser2 = argparse.ArgumentParser(add_help=False, parents=[parser1])
parser2.set_defaults(**defaults)
parser2.add_argument(
"--current-version", metavar="VERSION", help="Version that needs to be updated", required=False
)
parser2.add_argument(
"--parse",
metavar="REGEX",
help="Regex parsing the version string",
default="(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)",
)
parser2.add_argument(
"--serialize",
metavar="FORMAT",
help="How to format what is parsed back to a version",
default="{major}.{minor}.{patch}",
)
known_args, remaining_argv = parser2.parse_known_args(args)
defaults.update(vars(known_args))
time_context = {"now": datetime.now(), "utcnow": datetime.utcnow()}
v = Version(
known_args.parse,
known_args.serialize,
context=dict(list(time_context.items()) + list(prefixed_environ().items()) + list(vcs_info.items())),
)
if not "new_version" in defaults and known_args.current_version:
v.parse(known_args.current_version)
if len(positionals) > 0:
v.bump(positionals[0])
defaults["new_version"] = v.serialize()
parser3 = argparse.ArgumentParser(
description=DESCRIPTION,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
conflict_handler="resolve",
parents=[parser2],
)
parser3.set_defaults(**defaults)
parser3.add_argument(
"--current-version",
metavar="VERSION",
help="Version that needs to be updated",
required=not "current_version" in defaults,
)
parser3.add_argument(
"--dry-run", "-n", action="store_true", default=False, help="Don't write any files, just pretend."
)
parser3.add_argument(
"--new-version",
metavar="VERSION",
help="New version that should be in the files",
required=not "new_version" in defaults,
)
commitgroup = parser3.add_mutually_exclusive_group()
commitgroup.add_argument(
"--commit",
action="store_true",
dest="commit",
help="Commit to version control",
default=defaults.get("commit", False),
)
commitgroup.add_argument(
"--no-commit",
action="store_false",
dest="commit",
help="Do not commit to version control",
default=argparse.SUPPRESS,
)
taggroup = parser3.add_mutually_exclusive_group()
taggroup.add_argument(
"--tag",
action="store_true",
dest="tag",
default=defaults.get("tag", False),
help="Create a tag in version control",
)
taggroup.add_argument(
"--no-tag",
action="store_false",
dest="tag",
help="Do not create a tag in version control",
default=argparse.SUPPRESS,
)
parser3.add_argument(
"--tag-name",
metavar="TAG_NAME",
help="Tag name (only works with --tag)",
default=defaults.get("tag_name", "v{new_version}"),
)
parser3.add_argument(
"--message",
"-m",
metavar="COMMIT_MSG",
help="Commit message",
default=defaults.get("message", "Bump version: {current_version} → {new_version}"),
)
files = []
if "files" in defaults:
assert defaults["files"] != None
files = defaults["files"].split(" ")
parser3.add_argument("part", help="Part of the version to be bumped.")
parser3.add_argument("files", metavar="file", nargs="*", help="Files to change", default=files)
args = parser3.parse_args(remaining_argv + positionals)
files = files or positionals[1:]
for vcs in VCS:
if vcs.is_usable():
vcs.assert_nondirty()
break
# make sure files exist and contain version string
for path in files:
with io.open(path, "rb") as f:
before = f.read().decode("utf-8")
assert args.current_version in before, "Did not find string {} in file {}".format(args.current_version, path)
# change version string in files
for path in files:
with io.open(path, "rb") as f:
before = f.read().decode("utf-8")
# assert type(args.current_version) == bytes
# assert type(args.new_version) == bytes
after = before.replace(args.current_version, args.new_version)
if not args.dry_run:
with io.open(path, "wt", encoding="utf-8") as f:
f.write(after)
commit_files = files
if config:
config.remove_option("bumpversion", "new_version")
config.set("bumpversion", "current_version", args.new_version)
if not args.dry_run:
s = StringIO()
try:
config.write(s)
with io.open(known_args.config_file, "wb") as f:
f.write(s.getvalue().encode("utf-8"))
except UnicodeEncodeError:
warnings.warn(
"Unable to write UTF-8 to config file, because of an old configparser version. "
"Update with `pip install --upgrade configparser`."
)
commit_files.append(known_args.config_file)
if args.commit:
if not args.dry_run:
for path in commit_files:
vcs.add_path(path)
vcs_context = {"current_version": args.current_version, "new_version": args.new_version}
vcs_context.update(time_context)
vcs_context.update(prefixed_environ())
vcs.commit(message=args.message.format(**vcs_context))
if args.tag:
vcs.tag(args.tag_name.format(**vcs_context))
def main(original_args=None):
positionals, args = split_args_in_optional_and_positional(
sys.argv[1:] if original_args is None else original_args
)
parser1 = argparse.ArgumentParser(add_help=False)
parser1.add_argument(
'--config-file', default='.bumpversion.cfg', metavar='FILE',
help='Config file to read most of the variables from', required=False)
parser1.add_argument(
'--verbose', action='count', default=0,
help='Print verbose logging to stderr', required=False)
known_args, remaining_argv = parser1.parse_known_args(args)
if len(logger.handlers) == 0:
ch = logging.StreamHandler(sys.stderr)
logformatter = logging.Formatter('%(message)s')
ch.setFormatter(logformatter)
logger.addHandler(ch)
log_level = {
0: logging.WARNING,
1: logging.INFO,
2: logging.DEBUG,
}.get(known_args.verbose, logging.DEBUG)
logger.setLevel(log_level)
logger.debug("Starting {}".format(DESCRIPTION))
defaults = {}
vcs_info = {}
for vcs in VCS:
if vcs.is_usable():
vcs_info.update(vcs.latest_tag_info())
if 'current_version' in vcs_info:
defaults['current_version'] = vcs_info['current_version']
config = None
if os.path.exists(known_args.config_file):
config = RawConfigParser()
config.readfp(io.open(known_args.config_file, 'rt', encoding='utf-8'))
log_config = StringIO()
config.write(log_config)
logger.info("Reading config file {}:".format(known_args.config_file))
logger.info(log_config.getvalue())
defaults.update(dict(config.items("bumpversion")))
for listvaluename in ("serialize",):
try:
value = config.get("bumpversion", listvaluename)
defaults[listvaluename] = list(filter(None, (x.strip() for x in value.splitlines())))
except NoOptionError:
pass # no default value then ;)
for boolvaluename in ("commit", "tag", "dry_run"):
try:
defaults[boolvaluename] = config.getboolean(
"bumpversion", boolvaluename)
except NoOptionError:
pass # no default value then ;)
else:
message = "Could not read config file at {}".format(known_args.config_file)
if known_args.config_file != parser1.get_default('config_file'):
raise argparse.ArgumentTypeError(message)
else:
logger.info(message)
parser2 = argparse.ArgumentParser(add_help=False, parents=[parser1])
parser2.set_defaults(**defaults)
parser2.add_argument('--current-version', metavar='VERSION',
help='Version that needs to be updated', required=False)
parser2.add_argument('--parse', metavar='REGEX',
help='Regex parsing the version string',
default=defaults.get("parse", '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)'))
parser2.add_argument('--serialize', metavar='FORMAT',
action=DiscardDefaultIfSpecifiedAppendAction,
help='How to format what is parsed back to a version',
default=defaults.get("serialize", [str('{major}.{minor}.{patch}')]))
known_args, remaining_argv = parser2.parse_known_args(args)
defaults.update(vars(known_args))
assert type(known_args.serialize) == list
time_context = {
'now': datetime.now(),
'utcnow': datetime.utcnow(),
}
try:
v = Version(
known_args.parse,
known_args.serialize,
context=dict(list(time_context.items()) + list(prefixed_environ().items()) + list(vcs_info.items()))
)
except sre_constants.error as e:
sys.exit(1)
if not 'new_version' in defaults and known_args.current_version:
v.parse(known_args.current_version)
if len(positionals) > 0:
v.bump(positionals[0])
try:
defaults['new_version'] = v.serialize()
except MissingValueForSerializationException as e:
logger.info("Opportunistic finding of new_version failed: " + e.message)
except IncompleteVersionRepresenationException as e:
logger.info("Opportunistic finding of new_version failed: " + e.message)
except KeyError as e:
logger.info("Opportunistic finding of new_version failed")
parser3 = argparse.ArgumentParser(
description=DESCRIPTION,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
conflict_handler='resolve',
parents=[parser2],
)
parser3.set_defaults(**defaults)
parser3.add_argument('--current-version', metavar='VERSION',
help='Version that needs to be updated',
required=not 'current_version' in defaults)
parser3.add_argument('--dry-run', '-n', action='store_true',
default=False, help="Don't write any files, just pretend.")
parser3.add_argument('--new-version', metavar='VERSION',
help='New version that should be in the files',
required=not 'new_version' in defaults)
commitgroup = parser3.add_mutually_exclusive_group()
commitgroup.add_argument('--commit', action='store_true', dest="commit",
help='Commit to version control', default=defaults.get("commit", False))
commitgroup.add_argument('--no-commit', action='store_false', dest="commit",
help='Do not commit to version control', default=argparse.SUPPRESS)
taggroup = parser3.add_mutually_exclusive_group()
taggroup.add_argument('--tag', action='store_true', dest="tag", default=defaults.get("tag", False),
help='Create a tag in version control')
taggroup.add_argument('--no-tag', action='store_false', dest="tag",
help='Do not create a tag in version control', default=argparse.SUPPRESS)
parser3.add_argument('--tag-name', metavar='TAG_NAME',
help='Tag name (only works with --tag)',
default=defaults.get('tag_name', 'v{new_version}'))
parser3.add_argument('--message', '-m', metavar='COMMIT_MSG',
help='Commit message',
default=defaults.get('message', 'Bump version: {current_version} → {new_version}'))
files = []
if 'files' in defaults:
assert defaults['files'] != None
files = defaults['files'].split(' ')
parser3.add_argument('part',
help='Part of the version to be bumped.')
parser3.add_argument('files', metavar='file',
nargs='*',
help='Files to change', default=files)
args = parser3.parse_args(remaining_argv + positionals)
if args.dry_run:
logger.info("Dry run active, won't touch any files.")
logger.info("New version will be '{}'".format(args.new_version))
files = files or positionals[1:]
for vcs in VCS:
if vcs.is_usable():
vcs.assert_nondirty()
break
else:
vcs = None
# make sure files exist and contain version string
logger.info("Asserting files {} contain string '{}':".format(", ".join(files), args.current_version))
for path in files:
with io.open(path, 'rb') as f:
found_before = False
for lineno, line in enumerate(f.readlines()):
if args.current_version in line.decode('utf-8'):
found_before = True
logger.info("Found '{}' in {} at line {}: {}".format(args.current_version, path, lineno, line.decode('utf-8').rstrip()))
assert found_before, 'Did not find string {} in file {}'.format(
args.current_version, path)
# change version string in files
for path in files:
with io.open(path, 'rb') as f:
before = f.read().decode('utf-8')
after = before.replace(args.current_version, args.new_version)
logger.info("{} file {}:".format(
"Would change" if args.dry_run else "Changing",
path,
))
logger.info("\n".join(list(unified_diff(before.splitlines(), after.splitlines(), lineterm="", fromfile="a/"+path, tofile="b/"+path))))
if not args.dry_run:
with io.open(path, 'wt', encoding='utf-8') as f:
f.write(after)
commit_files = files
if config:
config.remove_option('bumpversion', 'new_version')
config.set('bumpversion', 'current_version', args.new_version)
s = StringIO()
try:
config.write(s)
logger.info("{} to config file {}:".format(
"Would write" if args.dry_run else "Writing",
known_args.config_file,
))
logger.info(log_config.getvalue())
if not args.dry_run:
with io.open(known_args.config_file, 'wb') as f:
f.write(s.getvalue().encode('utf-8'))
except UnicodeEncodeError:
warnings.warn(
"Unable to write UTF-8 to config file, because of an old configparser version. "
"Update with `pip install --upgrade configparser`."
)
commit_files.append(known_args.config_file)
if not vcs:
return
assert vcs.is_usable(), "Did find '{}' unusable, unable to commit.".format(vcs.__name__)
do_commit = (not args.dry_run) and args.commit
do_tag = (not args.dry_run) and args.tag
logger.info("{} {} commit".format(
"Would prepare" if not do_commit else "Preparing",
vcs.__name__,
))
for path in commit_files:
logger.info("{} changes in file '{}' to {}".format(
"Would add" if not do_commit else "Adding",
path,
vcs.__name__,
))
if do_commit:
vcs.add_path(path)
vcs_context = {
"current_version": args.current_version,
"new_version": args.new_version,
}
vcs_context.update(time_context)
vcs_context.update(prefixed_environ())
commit_message = args.message.format(**vcs_context)
logger.info("{} to {} with message '{}'".format(
"Would commit" if not do_commit else "Committing",
vcs.__name__,
commit_message,
))
if do_commit:
vcs.commit(message=commit_message)
tag_name = args.tag_name.format(**vcs_context)
logger.info("{} '{}' in {}".format(
"Would tag" if not do_tag else "Tagging",
tag_name,
vcs.__name__
))
if do_tag:
vcs.tag(tag_name)
def remove_option(self, option):
return _RawConfigParser.remove_option(self, self._default_section, option)
def main(args):
config = ConfigParser({"htrc": False,
"sentences": "False"})
config.read(args.config_file)
if config.getboolean("main", "sentences"):
from vsm.extensions.ldasentences import CorpusSent as Corpus
else:
from vsm.corpus import Corpus
if args.lang is None:
args.lang = []
args.corpus_path = config.get("main", "corpus_file")
c = Corpus.load(args.corpus_path)
# check for htrc metadata
if args.htrc or config.get("main", "htrc"):
htrc_langs = get_htrc_langs(args)
if htrc_langs:
args.lang.extend(new_langs)
# auto-guess a language
"""
new_langs = [lang for lang in detect_langs(c) if lang in langs and lang not in args.lang]
if new_langs:
args.lang.extend(new_langs)
"""
# add default locale if no other languages are specified
# do not add if in quiet mode -- make everything explicit
if not args.lang and not args.quiet:
import locale
locale = locale.getdefaultlocale()[0].split('_')[0].lower()
if locale in langs.keys():
args.lang.append(locale)
# check for any new candidates
args.lang = [lang for lang in args.lang if stop_language(c, langs[lang])]
if args.lang and not args.quiet:
args.lang = lang_prompt(args.lang)
stoplist = set()
# Apply stop words
print(" ")
for lang in args.lang:
print("Applying", langs[lang], "stopwords")
candidates = stop_language(c, langs[lang])
if len(candidates):
stoplist.update(candidates)
# Apply custom stopwords file
if args.stopword_file:
with open(args.stopword_file, encoding='utf8') as swf:
#candidates = [unidecode(word.strip()) for word in swf]
candidates = [word.strip() for word in swf]
if len(candidates):
print("Applying custom stopword file to remove {} word{}.".format(
len(candidates), 's' if len(candidates) > 1 else ''))
stoplist.update(candidates)
if args.min_word_len:
candidates = get_small_words(c, args.min_word_len)
if len(candidates):
print("Filtering {} small word{} with less than {} characters.".format(
len(candidates), 's' if len(candidates) > 1 else '', args.min_word_len))
stoplist.update(candidates)
if not args.special_chars:
candidates = get_special_chars(c)
if len(candidates):
print("Filtering {} word{} with special characters.".format(
len(candidates), 's' if len(candidates) > 1 else ''))
stoplist.update(candidates)
if args.high_filter is None and not args.quiet:
args.high_filter, candidates = get_high_filter(args, c, words=stoplist)
if len(candidates):
print("Filtering {} high frequency word{}.".format(len(candidates),
's' if len(candidates) > 1 else ''))
stoplist.update(candidates)
elif args.high_filter > 0:
candidates = get_candidate_words(c, args.high_filter, sort=False)
if len(candidates):
print("Filtering {} high frequency word{}.".format(len(candidates),
's' if len(candidates) > 1 else ''))
stoplist.update(candidates)
if args.low_filter is None and not args.quiet:
args.low_filter, candidates = get_low_filter(args, c, words=stoplist)
if len(candidates):
print("Filtering {} low frequency word{}.".format(len(candidates),
's' if len(candidates) > 1 else ''))
stoplist.update(candidates)
elif args.low_filter > 0:
candidates = get_candidate_words(c, -1 * args.low_filter, sort=False)
if len(candidates):
print("Filtering {} low frequency words.".format(len(candidates)))
stoplist.update(candidates)
if not stoplist:
print("No stopwords applied.\n\n")
sys.exit(0)
else:
print("\n\nApplying {} stopword{}".format(len(stoplist),
's' if len(stoplist) > 1 else ''))
c.in_place_stoplist(stoplist)
print("\n")
def name_corpus(dirname, languages, lowfreq=None, highfreq=None):
items, counts = get_items_counts(c.corpus)
corpus_name = [dirname]
if args.lang:
corpus_name.append('nltk')
corpus_name.append(''.join(args.lang))
if lowfreq > 0:
corpus_name.append('freq%s' % lowfreq)
else:
corpus_name.append('freq%s' % min(counts))
if highfreq > 0:
corpus_name.append('N%s' % highfreq)
else:
corpus_name.append('freq%s' % max(counts))
corpus_name = '-'.join(corpus_name)
corpus_name += '.npz'
return corpus_name
dirname = os.path.basename(args.corpus_path).split('-nltk-')[0].replace('.npz', '')
corpus_name = name_corpus(dirname, ['en'], args.low_filter, args.high_filter)
model_path = os.path.dirname(args.corpus_path)
args.corpus_path = os.path.join(model_path, corpus_name)
c.save(args.corpus_path)
config.set("main", "corpus_file", args.corpus_path)
config.remove_option("main", "model_pattern")
with open(args.config_file, 'w') as configfh:
config.write(configfh)
def main(original_args=None):
positionals, args = split_args_in_optional_and_positional(
sys.argv[1:] if original_args is None else original_args)
if len(positionals[1:]) > 2:
warnings.warn(
"Giving multiple files on the command line will be deprecated, please use [bumpversion:file:...] in a config file.",
PendingDeprecationWarning)
parser1 = argparse.ArgumentParser(add_help=False)
parser1.add_argument(
'--config-file',
metavar='FILE',
default=argparse.SUPPRESS,
required=False,
help=
'Config file to read most of the variables from (default: .bumpversion.cfg)'
)
parser1.add_argument('--verbose',
action='count',
default=0,
help='Print verbose logging to stderr',
required=False)
parser1.add_argument('--list',
action='store_true',
default=False,
help='List machine readable information',
required=False)
parser1.add_argument('--allow-dirty',
action='store_true',
default=False,
help="Don't abort if working directory is dirty",
required=False)
known_args, remaining_argv = parser1.parse_known_args(args)
logformatter = logging.Formatter('%(message)s')
if len(logger.handlers) == 0:
ch = logging.StreamHandler(sys.stderr)
ch.setFormatter(logformatter)
logger.addHandler(ch)
if len(logger_list.handlers) == 0:
ch2 = logging.StreamHandler(sys.stdout)
ch2.setFormatter(logformatter)
logger_list.addHandler(ch2)
if known_args.list:
logger_list.setLevel(1)
log_level = {
0: logging.WARNING,
1: logging.INFO,
2: logging.DEBUG,
}.get(known_args.verbose, logging.DEBUG)
logger.setLevel(log_level)
logger.debug("Starting {}".format(DESCRIPTION))
defaults = {}
vcs_info = {}
for vcs in VCS:
if vcs.is_usable():
vcs_info.update(vcs.latest_tag_info())
if 'current_version' in vcs_info:
defaults['current_version'] = vcs_info['current_version']
config = RawConfigParser('')
# don't transform keys to lowercase (which would be the default)
config.optionxform = lambda option: option
config.add_section('bumpversion')
explicit_config = hasattr(known_args, 'config_file')
if explicit_config:
config_file = known_args.config_file
elif not os.path.exists('.bumpversion.cfg') and \
os.path.exists('setup.cfg'):
config_file = 'setup.cfg'
else:
config_file = '.bumpversion.cfg'
config_file_exists = os.path.exists(config_file)
part_configs = {}
files = []
if config_file_exists:
logger.info("Reading config file {}:".format(config_file))
logger.info(io.open(config_file, 'rt', encoding='utf-8').read())
config.readfp(io.open(config_file, 'rt', encoding='utf-8'))
log_config = StringIO()
config.write(log_config)
if 'files' in dict(config.items("bumpversion")):
warnings.warn(
"'files =' configuration is will be deprecated, please use [bumpversion:file:...]",
PendingDeprecationWarning)
defaults.update(dict(config.items("bumpversion")))
for listvaluename in ("serialize", ):
try:
value = config.get("bumpversion", listvaluename)
defaults[listvaluename] = list(
filter(None, (x.strip() for x in value.splitlines())))
except NoOptionError:
pass # no default value then ;)
for boolvaluename in ("commit", "tag", "dry_run"):
try:
defaults[boolvaluename] = config.getboolean(
"bumpversion", boolvaluename)
except NoOptionError:
pass # no default value then ;)
for section_name in config.sections():
section_name_match = re.compile(
"^bumpversion:(file|part):(.+)").match(section_name)
if not section_name_match:
continue
section_prefix, section_value = section_name_match.groups()
section_config = dict(config.items(section_name))
if section_prefix == "part":
ThisVersionPartConfiguration = NumericVersionPartConfiguration
if 'values' in section_config:
section_config['values'] = list(
filter(
None,
(x.strip()
for x in section_config['values'].splitlines())))
ThisVersionPartConfiguration = ConfiguredVersionPartConfiguration
part_configs[section_value] = ThisVersionPartConfiguration(
**section_config)
elif section_prefix == "file":
filename = section_value
if 'serialize' in section_config:
section_config['serialize'] = list(
filter(
None,
(x.strip()
for x in section_config['serialize'].splitlines()
)))
section_config['part_configs'] = part_configs
if not 'parse' in section_config:
section_config['parse'] = defaults.get(
"parse",
'(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)')
if not 'serialize' in section_config:
section_config['serialize'] = defaults.get(
'serialize', [str('{major}.{minor}.{patch}')])
if not 'search' in section_config:
section_config['search'] = defaults.get(
"search", '{current_version}')
if not 'replace' in section_config:
section_config['replace'] = defaults.get(
"replace", '{new_version}')
files.append(
ConfiguredFile(filename, VersionConfig(**section_config)))
else:
message = "Could not read config file at {}".format(config_file)
if explicit_config:
raise argparse.ArgumentTypeError(message)
else:
logger.info(message)
parser2 = argparse.ArgumentParser(prog='bumpversion',
add_help=False,
parents=[parser1])
parser2.set_defaults(**defaults)
parser2.add_argument('--current-version',
metavar='VERSION',
help='Version that needs to be updated',
required=False)
parser2.add_argument('--parse',
metavar='REGEX',
help='Regex parsing the version string',
default=defaults.get(
"parse",
'(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)'))
parser2.add_argument('--serialize',
metavar='FORMAT',
action=DiscardDefaultIfSpecifiedAppendAction,
help='How to format what is parsed back to a version',
default=defaults.get(
"serialize", [str('{major}.{minor}.{patch}')]))
parser2.add_argument('--search',
metavar='SEARCH',
help='Template for complete string to search',
default=defaults.get("search", '{current_version}'))
parser2.add_argument('--replace',
metavar='REPLACE',
help='Template for complete string to replace',
default=defaults.get("replace", '{new_version}'))
known_args, remaining_argv = parser2.parse_known_args(args)
defaults.update(vars(known_args))
assert type(known_args.serialize) == list
context = dict(
list(time_context.items()) + list(prefixed_environ().items()) +
list(vcs_info.items()))
try:
vc = VersionConfig(
parse=known_args.parse,
serialize=known_args.serialize,
search=known_args.search,
replace=known_args.replace,
part_configs=part_configs,
)
except sre_constants.error as e:
sys.exit(1)
current_version = vc.parse(
known_args.current_version) if known_args.current_version else None
new_version = None
if not 'new_version' in defaults and known_args.current_version:
try:
if current_version and len(positionals) > 0:
logger.info("Attempting to increment part '{}'".format(
positionals[0]))
new_version = current_version.bump(positionals[0], vc.order())
logger.info("Values are now: " +
keyvaluestring(new_version._values))
defaults['new_version'] = vc.serialize(new_version, context)
except MissingValueForSerializationException as e:
logger.info("Opportunistic finding of new_version failed: " +
e.message)
except IncompleteVersionRepresenationException as e:
logger.info("Opportunistic finding of new_version failed: " +
e.message)
except KeyError as e:
logger.info("Opportunistic finding of new_version failed")
parser3 = argparse.ArgumentParser(
prog='bumpversion',
description=DESCRIPTION,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
conflict_handler='resolve',
parents=[parser2],
)
parser3.set_defaults(**defaults)
parser3.add_argument('--current-version',
metavar='VERSION',
help='Version that needs to be updated',
required=not 'current_version' in defaults)
parser3.add_argument('--dry-run',
'-n',
action='store_true',
default=False,
help="Don't write any files, just pretend.")
parser3.add_argument('--new-version',
metavar='VERSION',
help='New version that should be in the files',
required=not 'new_version' in defaults)
commitgroup = parser3.add_mutually_exclusive_group()
commitgroup.add_argument('--commit',
action='store_true',
dest="commit",
help='Commit to version control',
default=defaults.get("commit", False))
commitgroup.add_argument('--no-commit',
action='store_false',
dest="commit",
help='Do not commit to version control',
default=argparse.SUPPRESS)
taggroup = parser3.add_mutually_exclusive_group()
taggroup.add_argument('--tag',
action='store_true',
dest="tag",
default=defaults.get("tag", False),
help='Create a tag in version control')
taggroup.add_argument('--no-tag',
action='store_false',
dest="tag",
help='Do not create a tag in version control',
default=argparse.SUPPRESS)
signtagsgroup = parser3.add_mutually_exclusive_group()
signtagsgroup.add_argument('--sign-tags',
action='store_true',
dest="sign_tags",
help='Sign tags if created',
default=defaults.get("sign_tags", False))
signtagsgroup.add_argument('--no-sign-tags',
action='store_false',
dest="sign_tags",
help='Do not sign tags if created',
default=argparse.SUPPRESS)
parser3.add_argument('--tag-name',
metavar='TAG_NAME',
help='Tag name (only works with --tag)',
default=defaults.get('tag_name', 'v{new_version}'))
parser3.add_argument(
'--tag-message',
metavar='TAG_MESSAGE',
dest='tag_message',
help='Tag message',
default=defaults.get(
'tag_message', 'Bump version: {current_version} → {new_version}'))
parser3.add_argument(
'--message',
'-m',
metavar='COMMIT_MSG',
help='Commit message',
default=defaults.get(
'message', 'Bump version: {current_version} → {new_version}'))
file_names = []
if 'files' in defaults:
assert defaults['files'] != None
file_names = defaults['files'].split(' ')
parser3.add_argument('part', help='Part of the version to be bumped.')
parser3.add_argument('files',
metavar='file',
nargs='*',
help='Files to change',
default=file_names)
args = parser3.parse_args(remaining_argv + positionals)
if args.dry_run:
logger.info("Dry run active, won't touch any files.")
if args.new_version:
new_version = vc.parse(args.new_version)
logger.info("New version will be '{}'".format(args.new_version))
file_names = file_names or positionals[1:]
for file_name in file_names:
files.append(ConfiguredFile(file_name, vc))
for vcs in VCS:
if vcs.is_usable():
try:
vcs.assert_nondirty()
except WorkingDirectoryIsDirtyException as e:
if not defaults['allow_dirty']:
logger.warn(
"{}\n\nUse --allow-dirty to override this if you know what you're doing."
.format(e.message))
raise
break
else:
vcs = None
# make sure files exist and contain version string
logger.info("Asserting files {} contain the version string:".format(
", ".join([str(f) for f in files])))
for f in files:
f.should_contain_version(current_version, context)
# change version string in files
for f in files:
f.replace(current_version, new_version, context, args.dry_run)
commit_files = [f.path for f in files]
config.set('bumpversion', 'new_version', args.new_version)
for key, value in config.items('bumpversion'):
logger_list.info("{}={}".format(key, value))
config.remove_option('bumpversion', 'new_version')
config.set('bumpversion', 'current_version', args.new_version)
new_config = StringIO()
try:
write_to_config_file = (not args.dry_run) and config_file_exists
logger.info("{} to config file {}:".format(
"Would write" if not write_to_config_file else "Writing",
config_file,
))
config.write(new_config)
logger.info(new_config.getvalue())
if write_to_config_file:
with io.open(config_file, 'wb') as f:
f.write(new_config.getvalue().encode('utf-8'))
except UnicodeEncodeError:
warnings.warn(
"Unable to write UTF-8 to config file, because of an old configparser version. "
"Update with `pip install --upgrade configparser`.")
if config_file_exists:
commit_files.append(config_file)
if not vcs:
return
assert vcs.is_usable(), "Did find '{}' unusable, unable to commit.".format(
vcs.__name__)
do_commit = (not args.dry_run) and args.commit
do_tag = (not args.dry_run) and args.tag
logger.info("{} {} commit".format(
"Would prepare" if not do_commit else "Preparing",
vcs.__name__,
))
for path in commit_files:
logger.info("{} changes in file '{}' to {}".format(
"Would add" if not do_commit else "Adding",
path,
vcs.__name__,
))
if do_commit:
vcs.add_path(path)
vcs_context = {
"current_version": args.current_version,
"new_version": args.new_version,
}
vcs_context.update(time_context)
vcs_context.update(prefixed_environ())
commit_message = args.message.format(**vcs_context)
logger.info("{} to {} with message '{}'".format(
"Would commit" if not do_commit else "Committing",
vcs.__name__,
commit_message,
))
if do_commit:
vcs.commit(message=commit_message)
sign_tags = args.sign_tags
tag_name = args.tag_name.format(**vcs_context)
tag_message = args.tag_message.format(**vcs_context)
logger.info("{} '{}' {} in {} and {}".format(
"Would tag" if not do_tag else "Tagging", tag_name,
"with message '{}'".format(tag_message)
if tag_message else "without message", vcs.__name__,
"signing" if sign_tags else "not signing"))
if do_tag:
vcs.tag(sign_tags, tag_name, tag_message)
class Config:
"""A wrapper around RawConfigParser.
Provides a ``defaults`` attribute of the same type which can be used
to set default values.
"""
def __init__(self, version=None, _defaults=True):
"""Use read() to read in an existing config file.
version should be an int starting with 0 that gets incremented if you
want to register a new upgrade function. If None, upgrade is disabled.
"""
self._config = ConfigParser(dict_type=_sorted_dict)
self.defaults = None
if _defaults:
self.defaults = Config(_defaults=False)
self._version = version
self._loaded_version = None
self._upgrade_funcs = []
def _do_upgrade(self, func):
assert self._loaded_version is not None
assert self._version is not None
old_version = self._loaded_version
new_version = self._version
if old_version != new_version:
print_d("Config upgrade: %d->%d (%r)" %
(old_version, new_version, func))
func(self, old_version, new_version)
def get_version(self):
"""Get the version of the loaded config file (for testing only)
Raises Error if no file was loaded or versioning is disabled.
"""
if self._version is None:
raise Error("Versioning disabled")
if self._loaded_version is None:
raise Error("No file loaded")
return self._loaded_version
def register_upgrade_function(self, function):
"""Register an upgrade function that gets called at each read()
if the current config version and the loaded version don't match.
Can also be registered after read was called.
function(config, old_version: int, new_version: int) -> None
"""
if self._version is None:
raise Error("Versioning disabled")
self._upgrade_funcs.append(function)
# after read(), so upgrade now
if self._loaded_version is not None:
self._do_upgrade(function)
return function
def reset(self, section, option):
"""Reset the value to the default state"""
assert self.defaults is not None
try:
self._config.remove_option(section, option)
except NoSectionError:
pass
def options(self, section):
"""Returns a list of options available in the specified section."""
try:
options = self._config.options(section)
except NoSectionError:
if self.defaults:
return self.defaults.options(section)
raise
else:
if self.defaults:
try:
options.extend(self.defaults.options(section))
options = list_unique(options)
except NoSectionError:
pass
return options
def get(self, section, option, default=_DEFAULT):
"""get(section, option[, default]) -> str
If default is not given or set, raises Error in case of an error
"""
try:
return self._config.get(section, option)
except Error:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.get(section, option)
except Error:
pass
raise
return default
def gettext(self, *args, **kwargs):
value = self.get(*args, **kwargs)
# make sure there are no surrogates
value.encode("utf-8")
return value
def getbytes(self, section, option, default=_DEFAULT):
try:
value = self._config.get(section, option)
value = value.encode("utf-8", "surrogateescape")
return value
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getbytes(section, option)
except Error:
pass
raise Error(e)
return default
def getboolean(self, section, option, default=_DEFAULT):
"""getboolean(section, option[, default]) -> bool
If default is not given or set, raises Error in case of an error
"""
try:
return self._config.getboolean(section, option)
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getboolean(section, option)
except Error:
pass
raise Error(e)
return default
def getint(self, section, option, default=_DEFAULT):
"""getint(section, option[, default]) -> int
If default is not give or set, raises Error in case of an error
"""
try:
return int(self._config.getfloat(section, option))
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getint(section, option)
except Error:
pass
raise Error(e)
return default
def getfloat(self, section, option, default=_DEFAULT):
"""getfloat(section, option[, default]) -> float
If default is not give or set, raises Error in case of an error
"""
try:
return self._config.getfloat(section, option)
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getfloat(section, option)
except Error:
pass
raise Error(e)
return default
def getstringlist(self, section, option, default=_DEFAULT):
"""getstringlist(section, option[, default]) -> list
If default is not given or set, raises Error in case of an error.
Gets a list of strings, using CSV to parse and delimit.
"""
try:
value = self._config.get(section, option)
parser = csv.reader([value],
lineterminator='\n',
quoting=csv.QUOTE_MINIMAL)
try:
vals = next(parser)
except (csv.Error, ValueError) as e:
raise Error(e)
return vals
except Error as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getstringlist(section, option)
except Error:
pass
raise Error(e)
return default
def setstringlist(self, section, option, values):
"""Saves a list of unicode strings using the csv module"""
sw = StringIO()
values = [str(v) for v in values]
writer = csv.writer(sw, lineterminator='\n', quoting=csv.QUOTE_MINIMAL)
writer.writerow(values)
self.set(section, option, sw.getvalue())
def setlist(self, section, option, values, sep=","):
"""Saves a list of str using ',' as a separator and \\ for escaping"""
values = [str(v) for v in values]
joined = join_escape(values, sep)
self.set(section, option, joined)
def getlist(self, section, option, default=_DEFAULT, sep=","):
"""Returns a str list saved with setlist()"""
try:
value = self._config.get(section, option)
return split_escape(value, sep)
except (Error, ValueError) as e:
if default is _DEFAULT:
if self.defaults is not None:
try:
return self.defaults.getlist(section, option, sep=sep)
except Error:
pass
raise Error(e)
return default
def set(self, section, option, value):
"""Saves the string representation for the passed value
Don't pass unicode, encode first.
"""
if isinstance(value, bytes):
raise TypeError("use setbytes")
# RawConfigParser only allows string values but doesn't
# scream if they are not (and it only fails before the
# first config save..)
if not isinstance(value, str):
value = str(value)
try:
self._config.set(section, option, value)
except NoSectionError:
if self.defaults and self.defaults.has_section(section):
self._config.add_section(section)
self._config.set(section, option, value)
else:
raise
def settext(self, section, option, value):
value = str(value)
# make sure there are no surrogates
value.encode("utf-8")
self.set(section, option, value)
def setbytes(self, section, option, value):
assert isinstance(value, bytes)
value = value.decode("utf-8", "surrogateescape")
self.set(section, option, value)
def write(self, filename):
"""Write config to filename.
Can raise EnvironmentError
"""
assert isinstance(filename, fsnative)
mkdir(os.path.dirname(filename))
# temporary set the new version for saving
if self._version is not None:
self.add_section("__config__")
self.set("__config__", "version", self._version)
try:
with atomic_save(filename, "wb") as fileobj:
temp = StringIO()
self._config.write(temp)
data = temp.getvalue().encode("utf-8", "surrogateescape")
fileobj.write(data)
finally:
if self._loaded_version is not None:
self.set("__config__", "version", self._loaded_version)
def clear(self):
"""Remove all sections."""
for section in self._config.sections():
self._config.remove_section(section)
def is_empty(self):
"""Whether the config has any sections"""
return not self._config.sections()
def read(self, filename):
"""Reads the config from `filename` if the file exists,
otherwise does nothing
Can raise EnvironmentError, Error.
"""
try:
with open(filename, "rb") as fileobj:
fileobj = StringIO(fileobj.read().decode(
"utf-8", "surrogateescape"))
self._config.readfp(fileobj, filename)
except (IOError, OSError):
return
# don't upgrade if we just created a new config
if self._version is not None:
self._loaded_version = self.getint("__config__", "version", -1)
for func in self._upgrade_funcs:
self._do_upgrade(func)
def has_option(self, section, option):
"""If the given section exists, and contains the given option"""
return self._config.has_option(
section, option) or (self.defaults
and self.defaults.has_option(section, option))
def has_section(self, section):
"""If the given section exists"""
return self._config.has_section(section) or (
self.defaults and self.defaults.has_section(section))
def remove_option(self, section, option):
"""Remove the specified option from the specified section
Can raise Error.
"""
return self._config.remove_option(section, option)
def add_section(self, section):
"""Add a section named section to the instance if it not already
exists."""
if not self._config.has_section(section):
self._config.add_section(section)
def main(original_args=None):
positionals, args = split_args_in_optional_and_positional(
sys.argv[1:] if original_args is None else original_args
)
parser1 = argparse.ArgumentParser(add_help=False, prog='bumpversion')
parser1.add_argument(
'--config-file', default='.bumpversion.cfg', metavar='FILE',
help='Config file to read most of the variables from', required=False)
parser1.add_argument('-v', '--version', action='version',
version="%s %s" % (parser1.prog, __VERSION__))
known_args, remaining_argv = parser1.parse_known_args(args)
defaults = {}
vcs_info = {}
for vcs in VCS:
if vcs.is_usable():
vcs_info.update(vcs.latest_tag_info())
if 'current_version' in vcs_info:
defaults['current_version'] = vcs_info['current_version']
config = None
if os.path.exists(known_args.config_file):
config = RawConfigParser()
config.readfp(io.open(known_args.config_file, 'rt', encoding='utf-8'))
defaults.update(dict(config.items("bumpversion")))
for boolvaluename in ("commit", "tag", "dry_run"):
try:
defaults[boolvaluename] = config.getboolean(
"bumpversion", boolvaluename)
except NoOptionError:
pass # no default value then ;)
elif known_args.config_file != parser1.get_default('config_file'):
raise argparse.ArgumentTypeError("Could not read config file at {}".format(
known_args.config_file))
parser2 = argparse.ArgumentParser(add_help=False, parents=[parser1])
parser2.set_defaults(**defaults)
parser2.add_argument('--current-version', metavar='VERSION',
help='Version that needs to be updated', required=False)
parser2.add_argument('--parse', metavar='REGEX',
help='Regex parsing the version string',
default=defaults.get("parse", '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)'))
parser2.add_argument('--serialize', metavar='FORMAT',
help='How to format what is parsed back to a version',
default=defaults.get("serialize", str('{major}.{minor}.{patch}')))
known_args, remaining_argv = parser2.parse_known_args(args)
defaults.update(vars(known_args))
time_context = {
'now': datetime.now(),
'utcnow': datetime.utcnow(),
}
v = Version(
known_args.parse,
known_args.serialize,
context=dict(list(time_context.items()) + list(prefixed_environ().items()) + list(vcs_info.items()))
)
if not 'new_version' in defaults and known_args.current_version:
v.parse(known_args.current_version)
if len(positionals) > 0:
v.bump(positionals[0])
defaults['new_version'] = v.serialize()
parser3 = argparse.ArgumentParser(
description=DESCRIPTION,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
conflict_handler='resolve',
parents=[parser2],
)
parser3.set_defaults(**defaults)
parser3.add_argument('--current-version', metavar='VERSION',
help='Version that needs to be updated',
required=not 'current_version' in defaults)
parser3.add_argument('--dry-run', '-n', action='store_true',
default=False, help="Don't write any files, just pretend.")
parser3.add_argument('--new-version', metavar='VERSION',
help='New version that should be in the files',
required=not 'new_version' in defaults)
commitgroup = parser3.add_mutually_exclusive_group()
commitgroup.add_argument('--commit', action='store_true', dest="commit",
help='Commit to version control', default=defaults.get("commit", False))
commitgroup.add_argument('--no-commit', action='store_false', dest="commit",
help='Do not commit to version control', default=argparse.SUPPRESS)
taggroup = parser3.add_mutually_exclusive_group()
taggroup.add_argument('--tag', action='store_true', dest="tag", default=defaults.get("tag", False),
help='Create a tag in version control')
taggroup.add_argument('--no-tag', action='store_false', dest="tag",
help='Do not create a tag in version control', default=argparse.SUPPRESS)
parser3.add_argument('--tag-name', metavar='TAG_NAME',
help='Tag name (only works with --tag)',
default=defaults.get('tag_name', 'v{new_version}'))
parser3.add_argument('--message', '-m', metavar='COMMIT_MSG',
help='Commit message',
default=defaults.get('message', 'Bump version: {current_version} → {new_version}'))
files = []
if 'files' in defaults:
assert defaults['files'] != None
files = defaults['files'].split(' ')
parser3.add_argument('part',
help='Part of the version to be bumped.')
parser3.add_argument('files', metavar='file',
nargs='*',
help='Files to change', default=files)
args = parser3.parse_args(remaining_argv + positionals)
files = files or positionals[1:]
for vcs in VCS:
if vcs.is_usable():
vcs.assert_nondirty()
break
# make sure files exist and contain version string
for path in files:
with io.open(path, 'rb') as f:
before = f.read().decode('utf-8')
assert args.current_version in before, 'Did not find string {} in file {}'.format(
args.current_version, path)
# change version string in files
for path in files:
with io.open(path, 'rb') as f:
before = f.read().decode('utf-8')
# assert type(args.current_version) == bytes
# assert type(args.new_version) == bytes
after = before.replace(args.current_version, args.new_version)
if not args.dry_run:
with io.open(path, 'wt', encoding='utf-8') as f:
f.write(after)
commit_files = files
if config:
config.remove_option('bumpversion', 'new_version')
config.set('bumpversion', 'current_version', args.new_version)
if not args.dry_run:
s = StringIO()
try:
config.write(s)
with io.open(known_args.config_file, 'wb') as f:
f.write(s.getvalue().encode('utf-8'))
except UnicodeEncodeError:
warnings.warn(
"Unable to write UTF-8 to config file, because of an old configparser version. "
"Update with `pip install --upgrade configparser`."
)
commit_files.append(known_args.config_file)
if args.commit:
assert vcs.is_usable(), "Did find '{}' unusable, unable to commit.".format(vcs.__name__)
if not args.dry_run:
for path in commit_files:
vcs.add_path(path)
vcs_context = {
"current_version": args.current_version,
"new_version": args.new_version,
}
vcs_context.update(time_context)
vcs_context.update(prefixed_environ())
vcs.commit(message=args.message.format(**vcs_context))
if args.tag:
vcs.tag(args.tag_name.format(**vcs_context))
def main(args):
config = ConfigParser({"htrc": False, "sentences": "False"})
config.read(args.config_file)
if config.getboolean("main", "sentences"):
from vsm.extensions.ldasentences import CorpusSent as Corpus
else:
from vsm.corpus import Corpus
if args.lang is None:
args.lang = []
args.corpus_path = config.get("main", "corpus_file")
c = Corpus.load(args.corpus_path)
# check for htrc metadata
if args.htrc or config.get("main", "htrc"):
htrc_langs = get_htrc_langs(args)
if htrc_langs:
args.lang.extend(new_langs)
# auto-guess a language
"""
new_langs = [lang for lang in detect_langs(c) if lang in langs and lang not in args.lang]
if new_langs:
args.lang.extend(new_langs)
"""
# add default locale if no other languages are specified
# do not add if in quiet mode -- make everything explicit
if not args.lang and not args.quiet:
import locale
locale = locale.getdefaultlocale()[0].split('_')[0].lower()
if locale in langs.keys():
args.lang.append(locale)
# check for any new candidates
args.lang = [lang for lang in args.lang if stop_language(c, langs[lang])]
if args.lang and not args.quiet:
args.lang = lang_prompt(args.lang)
stoplist = set()
# Apply stop words
print(" ")
for lang in args.lang:
print("Applying", langs[lang], "stopwords")
candidates = stop_language(c, langs[lang])
if len(candidates):
stoplist.update(candidates)
# Apply custom stopwords file
if args.stopword_file:
with open(args.stopword_file, encoding='utf8') as swf:
#candidates = [unidecode(word.strip()) for word in swf]
candidates = [word.strip() for word in swf]
if len(candidates):
print("Applying custom stopword file to remove {} word{}.".
format(len(candidates),
's' if len(candidates) > 1 else ''))
stoplist.update(candidates)
if args.min_word_len:
candidates = get_small_words(c, args.min_word_len)
if len(candidates):
print("Filtering {} small word{} with less than {} characters.".
format(len(candidates), 's' if len(candidates) > 1 else '',
args.min_word_len))
stoplist.update(candidates)
if not args.special_chars:
candidates = get_special_chars(c)
if len(candidates):
print("Filtering {} word{} with special characters.".format(
len(candidates), 's' if len(candidates) > 1 else ''))
stoplist.update(candidates)
if args.high_filter is None and not args.quiet:
args.high_filter, candidates = get_high_filter(args, c, words=stoplist)
if len(candidates):
print("Filtering {} high frequency word{}.".format(
len(candidates), 's' if len(candidates) > 1 else ''))
stoplist.update(candidates)
elif args.high_filter > 0:
candidates = get_candidate_words(c, args.high_filter, sort=False)
if len(candidates):
print("Filtering {} high frequency word{}.".format(
len(candidates), 's' if len(candidates) > 1 else ''))
stoplist.update(candidates)
if args.low_filter is None and not args.quiet:
args.low_filter, candidates = get_low_filter(args, c, words=stoplist)
if len(candidates):
print("Filtering {} low frequency word{}.".format(
len(candidates), 's' if len(candidates) > 1 else ''))
stoplist.update(candidates)
elif args.low_filter > 0:
candidates = get_candidate_words(c, -1 * args.low_filter, sort=False)
if len(candidates):
print("Filtering {} low frequency words.".format(len(candidates)))
stoplist.update(candidates)
if not stoplist:
print("No stopwords applied.\n\n")
sys.exit(0)
else:
print("\n\nApplying {} stopword{}".format(
len(stoplist), 's' if len(stoplist) > 1 else ''))
c.in_place_stoplist(stoplist)
print("\n")
def name_corpus(dirname, languages, lowfreq=None, highfreq=None):
items, counts = get_items_counts(c.corpus)
corpus_name = [dirname]
if args.lang:
corpus_name.append('nltk')
corpus_name.append(''.join(args.lang))
if lowfreq > 0:
corpus_name.append('freq%s' % lowfreq)
else:
corpus_name.append('freq%s' % min(counts))
if highfreq > 0:
corpus_name.append('N%s' % highfreq)
else:
corpus_name.append('freq%s' % max(counts))
corpus_name = '-'.join(corpus_name)
corpus_name += '.npz'
return corpus_name
dirname = os.path.basename(args.corpus_path).split('-nltk-')[0].replace(
'.npz', '')
corpus_name = name_corpus(dirname, ['en'], args.low_filter,
args.high_filter)
model_path = os.path.dirname(args.corpus_path)
args.corpus_path = os.path.join(model_path, corpus_name)
c.save(args.corpus_path)
config.set("main", "corpus_file", args.corpus_path)
config.remove_option("main", "model_pattern")
with open(args.config_file, 'w') as configfh:
config.write(configfh)
class ParamStore(object):
def __init__(self, root_dir, file_name):
self._lock = threading.Lock()
with self._lock:
if not os.path.isdir(root_dir):
raise RuntimeError(
'Directory "' + root_dir + '" does not exist.')
self._path = os.path.join(root_dir, file_name)
self._dirty = False
# open config file
self._config = RawConfigParser()
self._config.read(self._path)
def flush(self):
if not self._dirty:
return
with self._lock:
self._dirty = False
with open(self._path, 'w') as of:
self._config.write(of)
def get(self, section, option, default=None):
"""Get a parameter value and return a string.
If default is specified and section or option are not defined
in the file, they are created and set to default, which is
then the return value.
"""
with self._lock:
if not self._config.has_option(section, option):
if default is not None:
self._set(section, option, default)
return default
return self._config.get(section, option)
def get_datetime(self, section, option, default=None):
result = self.get(section, option, default)
if result:
return WSDateTime.from_csv(result)
return result
def set(self, section, option, value):
"""Set option in section to string value."""
with self._lock:
self._set(section, option, value)
def _set(self, section, option, value):
if not self._config.has_section(section):
self._config.add_section(section)
elif (self._config.has_option(section, option) and
self._config.get(section, option) == value):
return
self._config.set(section, option, value)
self._dirty = True
def unset(self, section, option):
"""Remove option from section."""
with self._lock:
if not self._config.has_section(section):
return
if self._config.has_option(section, option):
self._config.remove_option(section, option)
self._dirty = True
if not self._config.options(section):
self._config.remove_section(section)
self._dirty = True
