def main():
# Read config settings
config = ConfigParser()
config.read('../config.cfg')
# Start logging
logging.basicConfig(level=config.get("LOGGING", "LEVEL"),
format="%(asctime)s - %(levelname)s - %(message)s")
logging.info("Started logging")
# Authenticating twitter API
logging.info("Authenticating Twitter API")
auth = OAuthHandler(config.get("TWITTER", "CONSUMER_KEY"),
config.get("TWITTER", "CONSUMER_SECRET"))
auth.set_access_token(config.get("TWITTER", "ACCESS_TOKEN"),
config.get("TWITTER", "ACCESS_TOKEN_SECRET"))
# Start streaming to AWS Kinesis
logging.info(
f"Start streaming tweets matching: {config.getlist('TWEEPY', 'TRACK_TOPICS')}"
)
twitter_stream = Stream(auth, TweetListener(config))
twitter_stream.filter(track=config.getlist("TWEEPY", "TRACK_TOPICS"),
languages=config.getlist("TWEEPY", "TRACK_LANGS"))
def __init__(self, config_file):
"""Read config files.
"""
parser = ConfigParser(warn)
parser.read((expanduser('~/.pypirc'), config_file))
main_section = 'mkrelease'
if not parser.has_section(main_section) and parser.has_section('defaults'):
main_section = 'defaults' # BBB
self.distbase = parser.getstring(main_section, 'distbase', '')
self.distdefault = parser.getlist(main_section, 'distdefault', [])
self.sign = parser.getboolean(main_section, 'sign', False)
self.identity = parser.getstring(main_section, 'identity', '')
self.push = parser.getboolean(main_section, 'push', False)
self.aliases = {}
if parser.has_section('aliases'):
for key, value in parser.items('aliases'):
self.aliases[key] = parser.to_list(value)
class ServerInfo(object):
def __init__(self, server):
self.sign = parser.getboolean(server, 'sign', None)
self.identity = parser.getstring(server, 'identity', None)
self.servers = {}
for server in parser.getlist('distutils', 'index-servers', []):
self.servers[server] = ServerInfo(server)
def extras_require() -> Mapping[str, Any]:
config = ConfigParser(
converters={"list": lambda x: fold(filter(None, x.split("\n")), [])})
config.read("setup.cfg")
return {
i: config.getlist("options.extras_require", i) # type: ignore
for i in config["options.extras_require"]
}
def get_ids() -> Dict[str, List[int]]:
"""Gets all database entries that are open in virtual windows
:return: a dictionary whose keys are databases names and values represent lists entries with open virtual windows
"""
check_vwm_config()
parser = ConfigParser(converters={'list': literal_eval})
parser.read(VWM_CFG_PATH)
options = parser.options('Windows')
ids_ = {}
for opt in options:
ids_[opt] = parser.getlist('Windows', opt)
return ids_
def remove_id(id_: int, database: str = None):
"""Removes the specified entry id from the list of open virtual windows
:param id_: an entry in the database
:param database: the name of the database
:return: dict of databases names and lists of entry ids with open virtual windows
"""
check_vwm_config()
if not database:
database = default_database()
parser = ConfigParser(converters={'list': literal_eval})
parser.read(VWM_CFG_PATH)
try:
ids_ = parser.getlist('Windows', database)
except NoOptionError:
parser.set('Windows', database, str([]))
ids_ = parser.getlist('Windows', database)
ids_.remove(id_)
parser.set('Windows', database, str(ids_))
with open(VWM_CFG_PATH, 'w') as file:
parser.write(file)
file.close()
return get_ids()
def add_id(journal_id: int, database: str = None):
"""Adds the specified entry id to the list of open virtual windows
:param journal_id: an entry in the database
:param database: the name of the database
:return: dict of databases names and lists of entry ids with open virtual windows
"""
check_vwm_config()
if not database:
database = default_database()
parser = ConfigParser(converters={'list': literal_eval})
parser.read(VWM_CFG_PATH)
try:
ids_ = parser.getlist('Windows', database)
except NoOptionError:
parser.set('Windows', database, str([]))
ids_ = parser.getlist('Windows', database)
if journal_id not in ids_: # TODO remove requirement to prevent double insertion
ids_.append(journal_id)
parser.set('Windows', database, str(ids_))
with open(VWM_CFG_PATH, 'w') as file:
parser.write(file)
file.close()
return get_ids()
def load_config():
def get_list(conf, section, option, **kwargs):
value = conf.get(section, option, **kwargs).strip()
return [x.strip() for x in value.split(",")] if value else []
parser = argparse.ArgumentParser(prog='file upload server')
parser.add_argument("--config",
help="configuration file location",
dest="config",
default="/etc/fus.conf")
args = parser.parse_args()
config = ConfigParser(interpolation=ExtendedInterpolation())
config.getlist = types.MethodType(get_list, config)
config.read(args.config)
# normalize basedir
config.set("global", "basedir",
os.path.abspath(config.get("global", "basedir")))
compute_permissions(config)
return config
def generate_gtpoints(config, phyreg_ids, min_area_sqkm, max_area_sqkm,
min_points, max_points):
'''
This function generates randomized points for ground truthing. It create
the GT field in the output shapefile.
Parameters
----------
config :
CanoPy configuration object
phyreg_ids : list
list of physiographic region IDs to process
min_area_sqkm : float
miminum area in square kilometers
max_area_sqkm : float
maximum area in square kilometers
min_points : int
minimum number of points allowed
max_points : int
maximum number of points allowed
'''
# fix user errors, if any
if min_area_sqkm > max_area_sqkm:
tmp = min_area_sqkm
min_area_sqkm = max_area_sqkm
max_area_sqkm = tmp
if min_points > max_points:
tmp = min_points
min_points = max_points
max_points = tmp
phyregs_layer = config.phyregs_layer
phyregs_area_sqkm_field = config.phyregs_area_sqkm_field
naipqq_layer = config.naipqq_layer
spatref_wkid = config.spatref_wkid
analysis_year = config.analysis_year
results_path = config.results_path
arcpy.env.overwriteOutput = True
arcpy.env.addOutputsToMap = False
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(spatref_wkid)
# use configparser converter to read list
conf = ConfigParser(
converters={'list': lambda x: [int(i.strip()) for i in x.split(',')]})
conf.read(config.config)
inverted_reg = conf.getlist('config', 'inverted_phyreg_ids')
# make sure to clear selection because most geoprocessing tools use
# selected features, if any
arcpy.SelectLayerByAttribute_management(naipqq_layer, 'CLEAR_SELECTION')
# select phyregs features to process
arcpy.SelectLayerByAttribute_management(phyregs_layer,
where_clause='PHYSIO_ID in (%s)' %
','.join(map(str, phyreg_ids)))
with arcpy.da.SearchCursor(
phyregs_layer,
['NAME', 'PHYSIO_ID', phyregs_area_sqkm_field]) as cur:
for row in cur:
name = row[0]
print(name)
# CreateRandomPoints cannot create a shapefile with - in its
# filename
name = name.replace(' ', '_').replace('-', '_')
phyreg_id = row[1]
area_sqkm = row[2]
# Check if region is inverted
if row[1] in inverted_reg:
inverted = True
else:
inverted = False
# +1 to count partial points; e.g., 0.1 requires one point
point_count = int(min_points + (max_points - min_points) /
(max_area_sqkm - min_area_sqkm) *
(area_sqkm - min_area_sqkm) + 1)
print('Raw point count: %d' % point_count)
if point_count < min_points:
point_count = min_points
elif point_count > max_points:
point_count = max_points
print('Final point count: %d' % point_count)
outdir_path = '%s/%s/Outputs' % (results_path, name)
shp_filename = 'gtpoints_%d_%s.shp' % (analysis_year, name)
tmp_shp_filename = 'tmp_%s' % shp_filename
tmp_shp_path = '%s/%s' % (outdir_path, tmp_shp_filename)
# create random points
arcpy.SelectLayerByAttribute_management(
phyregs_layer, where_clause='PHYSIO_ID=%d' % phyreg_id)
arcpy.CreateRandomPoints_management(outdir_path, tmp_shp_filename,
phyregs_layer, '', point_count)
# create a new field to store data for ground truthing
gt_field = 'GT'
arcpy.AddField_management(tmp_shp_path, gt_field, 'SHORT')
# spatially join the naip qq layer to random points to find
# output tile filenames
shp_path = '%s/%s' % (outdir_path, shp_filename)
arcpy.SpatialJoin_analysis(tmp_shp_path, naipqq_layer, shp_path)
# delete temporary point shapefile
arcpy.Delete_management(tmp_shp_path)
# get required fields from spatially joined point layer
with arcpy.da.UpdateCursor(
shp_path, ['SHAPE@XY', gt_field, 'FileName']) as cur2:
for row2 in cur2:
# read filename
filename = row2[2][:-13]
# construct the final output tile path
cfrtiffile_path = '%s/cfr%s.tif' % (outdir_path, filename)
# read the output tile as raster
ras = arcpy.sa.Raster(cfrtiffile_path)
# resolution
res = (ras.meanCellWidth, ras.meanCellHeight)
# convert raster to numpy array to read cell values
ras_a = arcpy.RasterToNumPyArray(ras)
# get xy values of point
xy = row2[0]
# perform calculate_row_column to get the row and column
# of the point
rc = config.__calculate_row_column(xy, ras.extent, res)
# update the point, correct inverted region points
if inverted is True:
row2[1] = 1 - ras_a[rc]
cur2.updateRow(row2)
else:
row2[1] = ras_a[rc]
cur.updateRow(row2)
# delete all fields except only those required
shp_desc = arcpy.Describe(shp_path)
oid_field = shp_desc.OIDFieldName
shape_field = shp_desc.shapeFieldName
all_fields = arcpy.ListFields(shp_path)
required_fields = [oid_field, shape_field, gt_field]
extra_fields = [
x.name for x in all_fields if x.name not in required_fields
]
arcpy.DeleteField_management(shp_path, extra_fields)
# clear selection again
arcpy.SelectLayerByAttribute_management(phyregs_layer, 'CLEAR_SELECTION')
print('Completed')
def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)
#
# ~/.weechat/weechat.conf 파싱
#
parser = ConfigParser(
strict=False,
empty_lines_in_values=False,
dict_type=ConfigParserMultiValues,
converters={ 'list': ConfigParserMultiValues.getlist },
)
parser.read(os.path.expanduser('~/.weechat/weechat.conf'))
lines = parser.getlist('layout', 'default.buffer')
p = re.compile(r'irc;(ozinger|uriirc)\.(#\w+?);')
channels = {
(r.group(1), r.group(2).lower())
for r in (p.search(e) for e in lines) if r
}
#
# 자주 들어가는 채널 검사
#
out = check_output(r'''
fd -tf '#' ~/.weechat/logs |
xargs -n 1 -P 8 sh -c 'echo $(rg "^2018.*\t@?(?:지현|지현_)\t" "$0" | wc -l) $0' |
sort -rn |
default=False,
action="store_true")
args = parser.parse_args()
#reading the configuration
config = ConfigParser(
converters={'list': lambda x: [str(i.strip()) for i in x.split(',')]})
config.read(args.cfg)
if args.dir == None: outdir = str(config.get("general", "outfolder"))
else: outdir = args.dir
outprefix = str(config.get("general", "folder_prefix"))
outfile = str(config.get("general", "outfile"))
models = config.getlist("eft", "models")
h_folder = "/rootFile"
vars_ = config.getlist("variables", "treenames")
#retrieve every subfolder for each operator
subf = glob(outdir + "/*/")
#cycle on the subfolders
print("@ ---- [INFO] Removing pure BSM ---- @")
for s in subf:
#retrieve process, ops from subfolder name
fol_name = s.split("/")[-2] # -1 is the /
process = fol_name.split(outprefix)[1]
process = process.strip(" ") # this is process + op
def _get_hook_from_config(
config: ConfigParser,
section: str,
hook_name: str,
suffix: str = "_command"
) -> Optional[Union[HookCommand, WebHookFactory]]:
"""Generate a HookCommand or WebHookFactory if one has been requested
within a section. They mostly share the same options."""
# Remember this will try to get from the DEFAULT section if not found
# and THEN use the fallback value.
full_hook_name = hook_name + suffix
cmd = None
url = None
if suffix == "_command":
cmd = config.getlist(section, full_hook_name, fallback=None)
else:
# This is the data payload part
cmd = config.get(section, full_hook_name, fallback=None)
url = config.get(section, full_hook_name + "_url", fallback=None)
if not url:
return
if not cmd:
return
if not config.getboolean(
section, full_hook_name + "_enabled", fallback=False):
return
logged = config.getboolean(section,
full_hook_name + "_logged",
fallback=False)
try:
allow_regex = _get_regex_from_config(section, config,
full_hook_name + "_allow_regex")
except EmptyRegexException:
logger.debug(
f"Empty value in [{section}] {full_hook_name + '_allow_regex'}")
allow_regex = None
try:
block_regex = _get_regex_from_config(section, config,
full_hook_name + "_block_regex")
except EmptyRegexException:
logger.debug(
f"Empty value in [{section}] {full_hook_name + '_block_regex'}")
block_regex = None
# allow_regex = None
# if allow_regex_str := config.get(
# section, hook_name + "_allow_regex", fallback=None):
# allow_regex = parse_regex_str(allow_regex_str)
# block_regex = None
# if block_regex_str := config.get(
# section, hook_name + "_block_regex", fallback=None):
# block_regex = parse_regex_str(block_regex_str)
if suffix == "_command" and isinstance(cmd, list):
hook = HookCommand(cmd=cmd,
logged=logged,
event_name=hook_name,
allow_regex=allow_regex,
block_regex=block_regex)
# The same object is used on many different videos, others are only used
# as part of the live stream object, so they are usually called once only anyway.
if hook_name in ("on_upcoming_detected", "on_video_detected"):
hook.call_only_once = False
return hook
if suffix == "_webhook":
hook = WebHookFactory(url=url,
payload=cmd,
logged=logged,
event_name=hook_name,
allow_regex=allow_regex,
block_regex=block_regex)
return hook
# https://tweepy.readthedocs.io/en/latest/streaming_how_to.html#handling-errors
if status_code == 420:
return False
if __name__ == "__main__":
# load config
config = ConfigParser(
converters={"list": lambda x: [e.strip() for e in x.split(",")]})
config.read("app.cfg")
# start logging
logging.basicConfig(level=config.get("logging", "level"),
format="%(asctime)s - %(levelname)s - %(message)s")
logging.info("Started")
# authenticate to twitter API
logging.info("Authenticating to Twitter")
auth = OAuthHandler(config.get("twitter", "consumer_key"),
config.get("twitter", "consumer_secret"))
auth.set_access_token(config.get("twitter", "access_token"),
config.get("twitter", "access_token_secret"))
# start streaming
logging.info(
f"Start streaming tweets matching: {config.getlist('tweepy', 'track_topics')}"
)
twitter_stream = Stream(auth, TweetListener(config))
twitter_stream.filter(track=config.getlist("tweepy", "track_topics"),
languages=config.getlist("tweepy", "track_langs"))
class Buildozer:
ERROR = 0
INFO = 1
DEBUG = 2
standard_cmds = ('distclean', 'update', 'debug', 'release',
'deploy', 'run', 'serve')
def __init__(self, filename='buildozer.spec', target=None):
self.log_level = 2
self.environ = {}
self.specfilename = filename
self.state = None
self.build_id = None
self.config_profile = ''
self.config = ConfigParser(allow_no_value=True)
self.config.optionxform = lambda value: value
self.config.getlist = self._get_config_list
self.config.getlistvalues = self._get_config_list_values
self.config.getdefault = self._get_config_default
self.config.getbooldefault = self._get_config_bool
self.config.getrawdefault = self._get_config_raw_default
if exists(filename):
self.config.read(filename, "utf-8")
self.check_configuration_tokens()
# Check all section/tokens for env vars, and replace the
# config value if a suitable env var exists.
set_config_from_envs(self.config)
try:
self.log_level = int(self.config.getdefault(
'buildozer', 'log_level', '2'))
except Exception:
pass
self.user_bin_dir = self.config.getdefault('buildozer', 'bin_dir', None)
if self.user_bin_dir:
self.user_bin_dir = realpath(join(self.root_dir, self.user_bin_dir))
self.targetname = None
self.target = None
if target:
self.set_target(target)
def set_target(self, target):
'''Set the target to use (one of buildozer.targets, such as "android")
'''
self.targetname = target
m = __import__('buildozer.targets.{0}'.format(target),
fromlist=['buildozer'])
self.target = m.get_target(self)
self.check_build_layout()
self.check_configuration_tokens()
def prepare_for_build(self):
'''Prepare the build.
'''
assert(self.target is not None)
if hasattr(self.target, '_build_prepared'):
return
self.info('Preparing build')
self.info('Check requirements for {0}'.format(self.targetname))
self.target.check_requirements()
self.info('Install platform')
self.target.install_platform()
self.info('Check application requirements')
self.check_application_requirements()
self.check_garden_requirements()
self.info('Compile platform')
self.target.compile_platform()
# flag to prevent multiple build
self.target._build_prepared = True
def build(self):
'''Do the build.
The target can set build_mode to 'release' or 'debug' before calling
this method.
(:meth:`prepare_for_build` must have been call before.)
'''
assert(self.target is not None)
assert(hasattr(self.target, '_build_prepared'))
if hasattr(self.target, '_build_done'):
return
# increment the build number
self.build_id = int(self.state.get('cache.build_id', '0')) + 1
self.state['cache.build_id'] = str(self.build_id)
self.info('Build the application #{}'.format(self.build_id))
self.build_application()
self.info('Package the application')
self.target.build_package()
# flag to prevent multiple build
self.target._build_done = True
#
# Log functions
#
def log(self, level, msg):
if level > self.log_level:
return
if USE_COLOR:
color = COLOR_SEQ(LOG_LEVELS_C[level])
print(''.join((RESET_SEQ, color, '# ', msg, RESET_SEQ)))
else:
print('{} {}'.format(LOG_LEVELS_T[level], msg))
def debug(self, msg):
self.log(self.DEBUG, msg)
def log_env(self, level, env):
"""dump env into debug logger in readable format"""
self.log(level, "ENVIRONMENT:")
for k, v in env.items():
self.log(level, " {} = {}".format(k, pformat(v)))
def info(self, msg):
self.log(self.INFO, msg)
def error(self, msg):
self.log(self.ERROR, msg)
#
# Internal check methods
#
def checkbin(self, msg, fn):
self.debug('Search for {0}'.format(msg))
if exists(fn):
return realpath(fn)
for dn in environ['PATH'].split(':'):
rfn = realpath(join(dn, fn))
if exists(rfn):
self.debug(' -> found at {0}'.format(rfn))
return rfn
self.error('{} not found, please install it.'.format(msg))
exit(1)
def cmd(self, command, **kwargs):
# prepare the environ, based on the system + our own env
env = copy(environ)
env.update(self.environ)
# prepare the process
kwargs.setdefault('env', env)
kwargs.setdefault('stdout', PIPE)
kwargs.setdefault('stderr', PIPE)
kwargs.setdefault('close_fds', True)
kwargs.setdefault('shell', True)
kwargs.setdefault('show_output', self.log_level > 1)
show_output = kwargs.pop('show_output')
get_stdout = kwargs.pop('get_stdout', False)
get_stderr = kwargs.pop('get_stderr', False)
break_on_error = kwargs.pop('break_on_error', True)
sensible = kwargs.pop('sensible', False)
run_condition = kwargs.pop('run_condition', None)
quiet = kwargs.pop('quiet', False)
if not quiet:
if not sensible:
self.debug('Run {0!r}'.format(command))
else:
if isinstance(command, (list, tuple)):
self.debug('Run {0!r} ...'.format(command[0]))
else:
self.debug('Run {0!r} ...'.format(command.split()[0]))
self.debug('Cwd {}'.format(kwargs.get('cwd')))
# open the process
if sys.platform == 'win32':
kwargs.pop('close_fds', None)
process = Popen(command, **kwargs)
# prepare fds
fd_stdout = process.stdout.fileno()
fd_stderr = process.stderr.fileno()
if fcntl:
fcntl.fcntl(
fd_stdout, fcntl.F_SETFL,
fcntl.fcntl(fd_stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
fcntl.fcntl(
fd_stderr, fcntl.F_SETFL,
fcntl.fcntl(fd_stderr, fcntl.F_GETFL) | os.O_NONBLOCK)
ret_stdout = [] if get_stdout else None
ret_stderr = [] if get_stderr else None
while not run_condition or run_condition():
try:
readx = select.select([fd_stdout, fd_stderr], [], [], 1)[0]
except select.error:
break
if fd_stdout in readx:
chunk = process.stdout.read()
if not chunk:
break
if get_stdout:
ret_stdout.append(chunk)
if show_output:
stdout.write(chunk.decode('utf-8', 'replace'))
if fd_stderr in readx:
chunk = process.stderr.read()
if not chunk:
break
if get_stderr:
ret_stderr.append(chunk)
if show_output:
stderr.write(chunk.decode('utf-8', 'replace'))
stdout.flush()
stderr.flush()
try:
process.communicate(
timeout=(1 if run_condition and not run_condition() else None)
)
except TimeoutExpired:
pass
if process.returncode != 0 and break_on_error:
self.error('Command failed: {0}'.format(command))
self.log_env(self.ERROR, kwargs['env'])
self.error('')
self.error('Buildozer failed to execute the last command')
if self.log_level <= self.INFO:
self.error('If the error is not obvious, please raise the log_level to 2')
self.error('and retry the latest command.')
else:
self.error('The error might be hidden in the log above this error')
self.error('Please read the full log, and search for it before')
self.error('raising an issue with buildozer itself.')
self.error('In case of a bug report, please add a full log with log_level = 2')
raise BuildozerCommandException()
if ret_stdout:
ret_stdout = b''.join(ret_stdout)
if ret_stderr:
ret_stderr = b''.join(ret_stderr)
return (ret_stdout.decode('utf-8', 'ignore') if ret_stdout else None,
ret_stderr.decode('utf-8') if ret_stderr else None,
process.returncode)
def cmd_expect(self, command, **kwargs):
from pexpect import spawnu
# prepare the environ, based on the system + our own env
env = copy(environ)
env.update(self.environ)
# prepare the process
kwargs.setdefault('env', env)
kwargs.setdefault('show_output', self.log_level > 1)
sensible = kwargs.pop('sensible', False)
show_output = kwargs.pop('show_output')
if show_output:
kwargs['logfile'] = codecs.getwriter('utf8')(stdout.buffer)
if not sensible:
self.debug('Run (expect) {0!r}'.format(command))
else:
self.debug('Run (expect) {0!r} ...'.format(command.split()[0]))
self.debug('Cwd {}'.format(kwargs.get('cwd')))
return spawnu(command, **kwargs)
def check_configuration_tokens(self):
'''Ensure the spec file is 'correct'.
'''
self.info('Check configuration tokens')
self.migrate_configuration_tokens()
get = self.config.getdefault
errors = []
adderror = errors.append
if not get('app', 'title', ''):
adderror('[app] "title" is missing')
if not get('app', 'source.dir', ''):
adderror('[app] "source.dir" is missing')
package_name = get('app', 'package.name', '')
if not package_name:
adderror('[app] "package.name" is missing')
elif package_name[0] in map(str, range(10)):
adderror('[app] "package.name" may not start with a number.')
version = get('app', 'version', '')
version_regex = get('app', 'version.regex', '')
if not version and not version_regex:
adderror('[app] One of "version" or "version.regex" must be set')
if version and version_regex:
adderror('[app] Conflict between "version" and "version.regex"'
', only one can be used.')
if version_regex and not get('app', 'version.filename', ''):
adderror('[app] "version.filename" is missing'
', required by "version.regex"')
orientation = get('app', 'orientation', 'landscape')
if orientation not in ('landscape', 'portrait', 'all', 'sensorLandscape'):
adderror('[app] "orientation" have an invalid value')
if errors:
self.error('{0} error(s) found in the buildozer.spec'.format(
len(errors)))
for error in errors:
print(error)
exit(1)
def migrate_configuration_tokens(self):
config = self.config
if config.has_section("app"):
migration = (
("android.p4a_dir", "p4a.source_dir"),
("android.p4a_whitelist", "android.whitelist"),
("android.bootstrap", "p4a.bootstrap"),
("android.branch", "p4a.branch"),
("android.p4a_whitelist_src", "android.whitelist_src"),
("android.p4a_blacklist_src", "android.blacklist_src")
)
for entry_old, entry_new in migration:
if not config.has_option("app", entry_old):
continue
value = config.get("app", entry_old)
config.set("app", entry_new, value)
config.remove_option("app", entry_old)
self.error("In section [app]: {} is deprecated, rename to {}!".format(
entry_old, entry_new))
def check_build_layout(self):
'''Ensure the build (local and global) directory layout and files are
ready.
'''
self.info('Ensure build layout')
if not exists(self.specfilename):
print('No {0} found in the current directory. Abandon.'.format(
self.specfilename))
exit(1)
# create global dir
self.mkdir(self.global_buildozer_dir)
self.mkdir(self.global_cache_dir)
# create local .buildozer/ dir
self.mkdir(self.buildozer_dir)
# create local bin/ dir
self.mkdir(self.bin_dir)
self.mkdir(self.applibs_dir)
self.state = JsonStore(join(self.buildozer_dir, 'state.db'))
target = self.targetname
if target:
self.mkdir(join(self.global_platform_dir, target, 'platform'))
self.mkdir(join(self.buildozer_dir, target, 'platform'))
self.mkdir(join(self.buildozer_dir, target, 'app'))
def check_application_requirements(self):
'''Ensure the application requirements are all available and ready to be
packaged as well.
'''
requirements = self.config.getlist('app', 'requirements', '')
target_available_packages = self.target.get_available_packages()
if target_available_packages is True:
# target handles all packages!
return
# remove all the requirements that the target can compile
onlyname = lambda x: x.split('==')[0] # noqa: E731
requirements = [x for x in requirements if onlyname(x) not in
target_available_packages]
if requirements and hasattr(sys, 'real_prefix'):
e = self.error
e('virtualenv is needed to install pure-Python modules, but')
e('virtualenv does not support nesting, and you are running')
e('buildozer in one. Please run buildozer outside of a')
e('virtualenv instead.')
exit(1)
# did we already installed the libs ?
if (
exists(self.applibs_dir) and
self.state.get('cache.applibs', '') == requirements
):
self.debug('Application requirements already installed, pass')
return
# recreate applibs
self.rmdir(self.applibs_dir)
self.mkdir(self.applibs_dir)
# ok now check the availability of all requirements
for requirement in requirements:
self._install_application_requirement(requirement)
# everything goes as expected, save this state!
self.state['cache.applibs'] = requirements
def _install_application_requirement(self, module):
self._ensure_virtualenv()
self.debug('Install requirement {} in virtualenv'.format(module))
self.cmd('pip install --target={} {}'.format(self.applibs_dir, module),
env=self.env_venv,
cwd=self.buildozer_dir)
def check_garden_requirements(self):
garden_requirements = self.config.getlist('app',
'garden_requirements', '')
if garden_requirements:
warnings.warn("`garden_requirements` settings is deprecated, use `requirements` instead", DeprecationWarning)
def _ensure_virtualenv(self):
if hasattr(self, 'venv'):
return
self.venv = join(self.buildozer_dir, 'venv')
if not self.file_exists(self.venv):
self.cmd('python3 -m venv ./venv',
cwd=self.buildozer_dir)
# read virtualenv output and parse it
output = self.cmd('bash -c "source venv/bin/activate && env"',
get_stdout=True,
cwd=self.buildozer_dir)
self.env_venv = copy(self.environ)
for line in output[0].splitlines():
args = line.split('=', 1)
if len(args) != 2:
continue
key, value = args
if key in ('VIRTUAL_ENV', 'PATH'):
self.env_venv[key] = value
if 'PYTHONHOME' in self.env_venv:
del self.env_venv['PYTHONHOME']
# ensure any sort of compilation will fail
self.env_venv['CC'] = '/bin/false'
self.env_venv['CXX'] = '/bin/false'
def mkdir(self, dn):
if exists(dn):
return
self.debug('Create directory {0}'.format(dn))
makedirs(dn)
def rmdir(self, dn):
if not exists(dn):
return
self.debug('Remove directory and subdirectory {}'.format(dn))
rmtree(dn)
def file_matches(self, patterns):
from glob import glob
result = []
for pattern in patterns:
matches = glob(expanduser(pattern.strip()))
result.extend(matches)
return result
def file_exists(self, *args):
return exists(join(*args))
def file_rename(self, source, target, cwd=None):
if cwd:
source = join(cwd, source)
target = join(cwd, target)
self.debug('Rename {0} to {1}'.format(source, target))
if not os.path.isdir(os.path.dirname(target)):
self.error(('Rename {0} to {1} fails because {2} is not a '
'directory').format(source, target, target))
move(source, target)
def file_copy(self, source, target, cwd=None):
if cwd:
source = join(cwd, source)
target = join(cwd, target)
self.debug('Copy {0} to {1}'.format(source, target))
copyfile(source, target)
def file_extract(self, archive, cwd=None):
if archive.endswith('.tgz') or archive.endswith('.tar.gz'):
self.cmd('tar xzf {0}'.format(archive), cwd=cwd)
return
if archive.endswith('.tbz2') or archive.endswith('.tar.bz2'):
# XXX same as before
self.cmd('tar xjf {0}'.format(archive), cwd=cwd)
return
if archive.endswith('.bin'):
# To process the bin files for linux and darwin systems
self.cmd('chmod a+x {0}'.format(archive), cwd=cwd)
self.cmd('./{0}'.format(archive), cwd=cwd)
return
if archive.endswith('.zip'):
self.cmd('unzip -q {}'.format(join(cwd, archive)), cwd=cwd)
return
raise Exception('Unhandled extraction for type {0}'.format(archive))
def file_copytree(self, src, dest):
print('copy {} to {}'.format(src, dest))
if os.path.isdir(src):
if not os.path.isdir(dest):
os.makedirs(dest)
files = os.listdir(src)
for f in files:
self.file_copytree(
os.path.join(src, f),
os.path.join(dest, f))
else:
copyfile(src, dest)
def clean_platform(self):
self.info('Clean the platform build directory')
if not exists(self.platform_dir):
return
rmtree(self.platform_dir)
def download(self, url, filename, cwd=None):
def report_hook(index, blksize, size):
if size <= 0:
progression = '{0} bytes'.format(index * blksize)
else:
progression = '{0:.2f}%'.format(
index * blksize * 100. / float(size))
if "CI" not in environ:
stdout.write('- Download {}\r'.format(progression))
stdout.flush()
url = url + filename
if cwd:
filename = join(cwd, filename)
if self.file_exists(filename):
unlink(filename)
self.debug('Downloading {0}'.format(url))
urlretrieve(url, filename, report_hook)
return filename
def get_version(self):
c = self.config
has_version = c.has_option('app', 'version')
has_regex = c.has_option('app', 'version.regex')
has_filename = c.has_option('app', 'version.filename')
# version number specified
if has_version:
if has_regex or has_filename:
raise Exception(
'version.regex and version.filename conflict with version')
return c.get('app', 'version')
# search by regex
if has_regex or has_filename:
if has_regex and not has_filename:
raise Exception('version.filename is missing')
if has_filename and not has_regex:
raise Exception('version.regex is missing')
fn = c.get('app', 'version.filename')
with open(fn) as fd:
data = fd.read()
regex = c.get('app', 'version.regex')
match = search(regex, data)
if not match:
raise Exception(
'Unable to find capture version in {0}\n'
' (looking for `{1}`)'.format(fn, regex))
version = match.groups()[0]
self.debug('Captured version: {0}'.format(version))
return version
raise Exception('Missing version or version.regex + version.filename')
def build_application(self):
self._copy_application_sources()
self._copy_application_libs()
self._add_sitecustomize()
def _copy_application_sources(self):
# XXX clean the inclusion/exclusion algo.
source_dir = realpath(expanduser(self.config.getdefault('app', 'source.dir', '.')))
include_exts = self.config.getlist('app', 'source.include_exts', '')
exclude_exts = self.config.getlist('app', 'source.exclude_exts', '')
exclude_dirs = self.config.getlist('app', 'source.exclude_dirs', '')
exclude_patterns = self.config.getlist('app', 'source.exclude_patterns', '')
include_patterns = self.config.getlist('app',
'source.include_patterns',
'')
app_dir = self.app_dir
include_exts = [ext.lower() for ext in include_exts]
exclude_exts = [ext.lower() for ext in exclude_exts]
exclude_dirs = [dir.lower() for dir in exclude_dirs]
exclude_patterns = [pat.lower() for pat in exclude_patterns]
include_patterns = [pat.lower() for pat in include_patterns]
self.debug('Copy application source from {}'.format(source_dir))
rmtree(self.app_dir)
for root, dirs, files in walk(source_dir, followlinks=True):
# avoid hidden directory
if True in [x.startswith('.') for x in root.split(sep)]:
continue
# need to have sort-of normalization. Let's say you want to exclude
# image directory but not images, the filtered_root must have a / at
# the end, same for the exclude_dir. And then we can safely compare
filtered_root = root[len(source_dir) + 1:].lower()
if filtered_root:
filtered_root += '/'
# manual exclude_dirs approach
is_excluded = False
for exclude_dir in exclude_dirs:
if exclude_dir[-1] != '/':
exclude_dir += '/'
if filtered_root.startswith(exclude_dir):
is_excluded = True
break
# pattern matching
if not is_excluded:
# match pattern if not ruled out by exclude_dirs
for pattern in exclude_patterns:
if fnmatch(filtered_root, pattern):
is_excluded = True
break
for pattern in include_patterns:
if fnmatch(filtered_root, pattern):
is_excluded = False
break
if is_excluded:
continue
for fn in files:
# avoid hidden files
if fn.startswith('.'):
continue
# pattern matching
is_excluded = False
dfn = fn.lower()
if filtered_root:
dfn = join(filtered_root, fn)
for pattern in exclude_patterns:
if fnmatch(dfn, pattern):
is_excluded = True
break
for pattern in include_patterns:
if fnmatch(dfn, pattern):
is_excluded = False
break
if is_excluded:
continue
# filter based on the extension
# TODO more filters
basename, ext = splitext(fn)
if ext:
ext = ext[1:].lower()
if include_exts and ext not in include_exts:
continue
if exclude_exts and ext in exclude_exts:
continue
sfn = join(root, fn)
rfn = realpath(join(app_dir, root[len(source_dir) + 1:], fn))
# ensure the directory exists
dfn = dirname(rfn)
self.mkdir(dfn)
# copy!
self.debug('Copy {0}'.format(sfn))
copyfile(sfn, rfn)
def _copy_application_libs(self):
# copy also the libs
copytree(self.applibs_dir, join(self.app_dir, '_applibs'))
def _add_sitecustomize(self):
copyfile(join(dirname(__file__), 'sitecustomize.py'),
join(self.app_dir, 'sitecustomize.py'))
main_py = join(self.app_dir, 'service', 'main.py')
if not self.file_exists(main_py):
return
header = (b'import sys, os; '
b'sys.path = [os.path.join(os.getcwd(),'
b'"..", "_applibs")] + sys.path\n')
with open(main_py, 'rb') as fd:
data = fd.read()
data = header + data
with open(main_py, 'wb') as fd:
fd.write(data)
self.info('Patched service/main.py to include applibs')
def namify(self, name):
'''Return a "valid" name from a name with lot of invalid chars
(allowed characters: a-z, A-Z, 0-9, -, _)
'''
return re.sub(r'[^a-zA-Z0-9_\-]', '_', name)
@property
def root_dir(self):
return realpath(expanduser(dirname(self.specfilename)))
@property
def user_build_dir(self):
"""The user-provided build dir, if any."""
# Check for a user-provided build dir
# Check the (deprecated) builddir token, for backwards compatibility
build_dir = self.config.getdefault('buildozer', 'builddir', None)
if build_dir is not None:
# for backwards compatibility, append .buildozer to builddir
build_dir = join(build_dir, '.buildozer')
build_dir = self.config.getdefault('buildozer', 'build_dir', build_dir)
if build_dir is not None:
build_dir = realpath(join(self.root_dir, expanduser(build_dir)))
return build_dir
@property
def buildozer_dir(self):
'''The directory in which to run the app build.'''
if self.user_build_dir is not None:
return self.user_build_dir
return join(self.root_dir, '.buildozer')
@property
def bin_dir(self):
if self.user_bin_dir:
return self.user_bin_dir
return join(self.root_dir, 'bin')
@property
def platform_dir(self):
return join(self.buildozer_dir, self.targetname, 'platform')
@property
def app_dir(self):
return join(self.buildozer_dir, self.targetname, 'app')
@property
def applibs_dir(self):
return join(self.buildozer_dir, 'applibs')
@property
def global_buildozer_dir(self):
return join(expanduser('~'), '.buildozer')
@property
def global_platform_dir(self):
return join(self.global_buildozer_dir, self.targetname, 'platform')
@property
def global_packages_dir(self):
return join(self.global_buildozer_dir, self.targetname, 'packages')
@property
def global_cache_dir(self):
return join(self.global_buildozer_dir, 'cache')
@property
def package_full_name(self):
package_name = self.config.getdefault('app', 'package.name', '')
package_domain = self.config.getdefault('app', 'package.domain', '')
if package_domain == '':
return package_name
return '{}.{}'.format(package_domain, package_name)
#
# command line invocation
#
def targets(self):
for fn in listdir(join(dirname(__file__), 'targets')):
if fn.startswith('.') or fn.startswith('__'):
continue
if not fn.endswith('.py'):
continue
target = fn[:-3]
try:
m = __import__('buildozer.targets.{0}'.format(target),
fromlist=['buildozer'])
yield target, m
except NotImplementedError:
pass
except:
raise
pass
def usage(self):
print('Usage:')
print(' buildozer [--profile <name>] [--verbose] [target] <command>...')
print(' buildozer --version')
print('')
print('Available targets:')
targets = list(self.targets())
for target, m in targets:
try:
doc = m.__doc__.strip().splitlines()[0].strip()
except Exception:
doc = '<no description>'
print(' {0:<18} {1}'.format(target, doc))
print('')
print('Global commands (without target):')
cmds = [x for x in dir(self) if x.startswith('cmd_')]
for cmd in cmds:
name = cmd[4:]
meth = getattr(self, cmd)
if not meth.__doc__:
continue
doc = [x for x in
meth.__doc__.strip().splitlines()][0].strip()
print(' {0:<18} {1}'.format(name, doc))
print('')
print('Target commands:')
print(' clean Clean the target environment')
print(' update Update the target dependencies')
print(' debug Build the application in debug mode')
print(' release Build the application in release mode')
print(' deploy Deploy the application on the device')
print(' run Run the application on the device')
print(' serve Serve the bin directory via SimpleHTTPServer')
for target, m in targets:
mt = m.get_target(self)
commands = mt.get_custom_commands()
if not commands:
continue
print('')
print('Target "{0}" commands:'.format(target))
for command, doc in commands:
if not doc:
continue
doc = textwrap.fill(textwrap.dedent(doc).strip(), 59,
subsequent_indent=' ' * 21)
print(' {0:<18} {1}'.format(command, doc))
print('')
def run_default(self):
self.check_build_layout()
if 'buildozer:defaultcommand' not in self.state:
print('No default command set.')
print('Use "buildozer setdefault <command args...>"')
print('Use "buildozer help" for a list of all commands"')
exit(1)
cmd = self.state['buildozer:defaultcommand']
self.run_command(cmd)
def run_command(self, args):
while args:
if not args[0].startswith('-'):
break
arg = args.pop(0)
if arg in ('-v', '--verbose'):
self.log_level = 2
elif arg in ('-h', '--help'):
self.usage()
exit(0)
elif arg in ('-p', '--profile'):
self.config_profile = args.pop(0)
elif arg == '--version':
print('Buildozer {0}'.format(__version__))
exit(0)
self._merge_config_profile()
self.check_root()
if not args:
self.run_default()
return
command, args = args[0], args[1:]
cmd = 'cmd_{0}'.format(command)
# internal commands ?
if hasattr(self, cmd):
getattr(self, cmd)(*args)
return
# maybe it's a target?
targets = [x[0] for x in self.targets()]
if command not in targets:
print('Unknown command/target {}'.format(command))
exit(1)
self.set_target(command)
self.target.run_commands(args)
def check_root(self):
'''If effective user id is 0, display a warning and require
user input to continue (or to cancel)'''
warn_on_root = self.config.getdefault('buildozer', 'warn_on_root', '1')
try:
euid = os.geteuid() == 0
except AttributeError:
if sys.platform == 'win32':
import ctypes
euid = ctypes.windll.shell32.IsUserAnAdmin() != 0
if warn_on_root == '1' and euid:
print('\033[91m\033[1mBuildozer is running as root!\033[0m')
print('\033[91mThis is \033[1mnot\033[0m \033[91mrecommended, and may lead to problems later.\033[0m')
cont = None
while cont not in ('y', 'n'):
cont = input('Are you sure you want to continue [y/n]? ')
if cont == 'n':
sys.exit()
def cmd_init(self, *args):
'''Create a initial buildozer.spec in the current directory
'''
if exists('buildozer.spec'):
print('ERROR: You already have a buildozer.spec file.')
exit(1)
copyfile(join(dirname(__file__), 'default.spec'), 'buildozer.spec')
print('File buildozer.spec created, ready to customize!')
def cmd_distclean(self, *args):
'''Clean the whole Buildozer environment.
'''
print("Warning: Your ndk, sdk and all other cached packages will be"
" removed. Continue? (y/n)")
if sys.stdin.readline().lower()[0] == 'y':
self.info('Clean the global build directory')
if not exists(self.global_buildozer_dir):
return
rmtree(self.global_buildozer_dir)
def cmd_appclean(self, *args):
'''Clean the .buildozer folder in the app directory.
This command specifically refuses to delete files in a
user-specified build directory, to avoid accidentally deleting
more than the user intends.
'''
if self.user_build_dir is not None:
self.error(
('Failed: build_dir is specified as {} in the buildozer config. `appclean` will '
'not attempt to delete files in a user-specified build directory.').format(self.user_build_dir))
elif exists(self.buildozer_dir):
self.info('Deleting {}'.format(self.buildozer_dir))
rmtree(self.buildozer_dir)
else:
self.error('{} already deleted, skipping.'.format(self.buildozer_dir))
def cmd_help(self, *args):
'''Show the Buildozer help.
'''
self.usage()
def cmd_setdefault(self, *args):
'''Set the default command to run when no arguments are given
'''
self.check_build_layout()
self.state['buildozer:defaultcommand'] = args
def cmd_version(self, *args):
'''Show the Buildozer version
'''
print('Buildozer {0}'.format(__version__))
def cmd_serve(self, *args):
'''Serve the bin directory via SimpleHTTPServer
'''
try:
from http.server import SimpleHTTPRequestHandler
from socketserver import TCPServer
except ImportError:
from SimpleHTTPServer import SimpleHTTPRequestHandler
from SocketServer import TCPServer
os.chdir(self.bin_dir)
handler = SimpleHTTPRequestHandler
httpd = TCPServer(("", SIMPLE_HTTP_SERVER_PORT), handler)
print("Serving via HTTP at port {}".format(SIMPLE_HTTP_SERVER_PORT))
print("Press Ctrl+c to quit serving.")
httpd.serve_forever()
#
# Private
#
def _merge_config_profile(self):
profile = self.config_profile
if not profile:
return
for section in self.config.sections():
# extract the profile part from the section name
# example: [app@default,hd]
parts = section.split('@', 1)
if len(parts) < 2:
continue
# create a list that contain all the profiles of the current section
# ['default', 'hd']
section_base, section_profiles = parts
section_profiles = section_profiles.split(',')
if profile not in section_profiles:
continue
# the current profile is one available in the section
# merge with the general section, or make it one.
if not self.config.has_section(section_base):
self.config.add_section(section_base)
for name, value in self.config.items(section):
print('merged ({}, {}) into {} (profile is {})'.format(name,
value, section_base, profile))
self.config.set(section_base, name, value)
def _get_config_list_values(self, *args, **kwargs):
kwargs['with_values'] = True
return self._get_config_list(*args, **kwargs)
def _get_config_list(self, section, token, default=None, with_values=False):
# monkey-patch method for ConfigParser
# get a key as a list of string, separated from the comma
# check if an env var exists that should replace the file config
set_config_token_from_env(section, token, self.config)
# if a section:token is defined, let's use the content as a list.
l_section = '{}:{}'.format(section, token)
if self.config.has_section(l_section):
values = self.config.options(l_section)
if with_values:
return ['{}={}'.format(key, self.config.get(l_section, key)) for
key in values]
else:
return [x.strip() for x in values]
values = self.config.getdefault(section, token, '')
if not values:
return default
values = values.split(',')
if not values:
return default
return [x.strip() for x in values]
def _get_config_default(self, section, token, default=None):
# monkey-patch method for ConfigParser
# get an appropriate env var if it exists, else
# get a key in a section, or the default
# check if an env var exists that should replace the file config
set_config_token_from_env(section, token, self.config)
if not self.config.has_section(section):
return default
if not self.config.has_option(section, token):
return default
return self.config.get(section, token)
def _get_config_bool(self, section, token, default=False):
# monkey-patch method for ConfigParser
# get a key in a section, or the default
# check if an env var exists that should replace the file config
set_config_token_from_env(section, token, self.config)
if not self.config.has_section(section):
return default
if not self.config.has_option(section, token):
return default
return self.config.getboolean(section, token)
def _get_config_raw_default(self, section, token, default=None, section_sep="=", split_char=" "):
l_section = '{}:{}'.format(section, token)
if self.config.has_section(l_section):
return [section_sep.join(item) for item in self.config.items(l_section)]
if not self.config.has_option(section, token):
return default.split(split_char)
return self.config.get(section, token).split(split_char)
required=False,
default=1,
type=int)
args = parser.parse_args()
ROOT.gROOT.SetBatch(1)
config = ConfigParser(
converters={'list': lambda x: [str(i.strip()) for i in x.split(',')]})
config.read(args.cfg)
outdir = config.get("general", "outfolder")
outprefix = config.get("general", "folder_prefix")
outfile = config.get("general", "outfile")
models = config.getlist("eft", "models")
fillMissing_ = 0
if config.has_option("eft", "fillMissing"):
fillMissing_ = config.get("eft", "fillMissing")
opr = createOpRange(config)
mkdir(outdir)
makeActivations(
outdir, config
) #make scripts for automatic activation of text2workspace and combine
if len(models) == 0: sys.exit("[ERROR] No model specified, exiting ...")
if config.has_option("files", "module"):
exec("import Utils.{} as fileRetriever".format(
def update_gtpoints(config, old_points, phyreg_ids):
'''
This function copies a previous years GT points and copies the
points but with the new years GT values. It addtionally corrects the
values if they are within an inverted region.
Parameters
----------
config :
CanoPy configuration object
old_points : str
Layer name for the previous years points
phyreg_ids : list
list of physiographic region IDs to process
'''
phyregs_layer = config.phyregs_layer
naipqq_layer = config.naipqq_layer
spatref_wkid = config.spatref_wkid
analysis_year = config.analysis_year
results_path = config.results_path
arcpy.env.overwriteOutput = True
arcpy.env.addOutputsToMap = False
arcpy.env.outputCoordinateSystem = arcpy.SpatialReference(spatref_wkid)
# use configparser converter to read list
conf = ConfigParser(
converters={'list': lambda x: [int(i.strip()) for i in x.split(',')]})
conf.read(config.config)
inverted_reg = conf.getlist('config', 'inverted_phyreg_ids')
# make sure to clear selection because most geoprocessing tools use
# selected features, if any
arcpy.SelectLayerByAttribute_management(naipqq_layer, 'CLEAR_SELECTION')
# select phyregs features to process
arcpy.SelectLayerByAttribute_management(phyregs_layer,
where_clause='PHYSIO_ID in (%s)' %
','.join(map(str, phyreg_ids)))
with arcpy.da.SearchCursor(phyregs_layer, ['NAME', 'PHYSIO_ID']) as cur:
for row in cur:
name = row[0]
print(name)
# CreateRandomPoints cannot create a shapefile with - in its
# filename
name = name.replace(' ', '_').replace('-', '_')
phyreg_id = row[1]
# Check if region is inverted
if row[1] in inverted_reg:
inverted = True
else:
inverted = False
outdir_path = '%s/%s/Outputs' % (results_path, name)
shp_filename = 'gtpoints_%d_%s.shp' % (analysis_year, name)
tmp_shp_filename = 'tmp_%s' % shp_filename
tmp_shp_path = '%s/%s' % (outdir_path, tmp_shp_filename)
# create random points
arcpy.SelectLayerByAttribute_management(
phyregs_layer, where_clause='PHYSIO_ID=%d' % phyreg_id)
# create a new field to store data for ground truthing
gt_field = 'GT_%s' % analysis_year
arcpy.CopyFeatures_management(old_points, tmp_shp_path)
arcpy.AddField_management(tmp_shp_path, gt_field, 'SHORT')
# spatially join the naip qq layer to random points to find
# output tile filenames
shp_path = '%s/%s' % (outdir_path, shp_filename)
arcpy.SpatialJoin_analysis(tmp_shp_path, naipqq_layer, shp_path)
# delete temporary point shapefile
arcpy.Delete_management(tmp_shp_path)
# get required fields from spatially joined point layer
with arcpy.da.UpdateCursor(
shp_path, ['SHAPE@XY', gt_field, 'FileName']) as cur2:
for row2 in cur2:
# read filename
filename = row2[2][:-13]
# construct the final output tile path
cfrtiffile_path = '%s/cfr%s.tif' % (outdir_path, filename)
# read the output tile as raster
ras = arcpy.sa.Raster(cfrtiffile_path)
# resolution
res = (ras.meanCellWidth, ras.meanCellHeight)
# convert raster to numpy array to read cell values
ras_a = arcpy.RasterToNumPyArray(ras)
# get xy values of point
xy = row2[0]
# perform calculate_row_column to get the row and column
# of the point
rc = __calculate_row_column(xy, ras.extent, res)
# update the point, correct inverted region points
if inverted is True:
row2[1] = 1 - ras_a[rc]
cur2.updateRow(row2)
else:
row2[1] = ras_a[rc]
cur2.updateRow(row2)
# delete all fields except only those required
shp_desc = arcpy.Describe(shp_path)
oid_field = shp_desc.OIDFieldName
shape_field = shp_desc.shapeFieldName
all_fields = arcpy.ListFields(shp_path)
required_fields = [oid_field, shape_field, gt_field]
extra_fields = [
x.name for x in all_fields if x.name not in required_fields
]
arcpy.DeleteField_management(shp_path, extra_fields)
# clear selection again
arcpy.SelectLayerByAttribute_management(phyregs_layer, 'CLEAR_SELECTION')
print('Completed')
def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)
#
# ~/.weechat/weechat.conf 파싱
#
parser = ConfigParser(
strict=False,
empty_lines_in_values=False,
dict_type=ConfigParserMultiValues,
converters={'list': ConfigParserMultiValues.getlist},
)
parser.read(os.path.expanduser('~/.weechat/weechat.conf'))
lines = parser.getlist('layout', 'default.buffer')
p = re.compile(r'irc;(ozinger|uriirc)\.(#\w+?);')
channels = {(r.group(1), r.group(2).lower())
for r in (p.search(e) for e in lines) if r}
#
# 자주 들어가는 채널 검사
#
out = check_output(r'''
fd -tf '#' ~/.weechat/logs |
xargs -n 1 -P 8 sh -c 'echo $(rg "^2018.*\t@?(?:지현|지현_)\t" "$0" | wc -l) $0' |
sort -rn |
awk '{print $2}'
''',
shell=True,
parser.add_argument("--exthelp",
help="extended help",
dest="exthelp",
action="store_true")
args = parser.parse_known_args()[0]
conf_file = args.config
if not os.access(conf_file, os.R_OK):
logging.getLogger(__name__).error("Please provide a config file")
sys.exit(1)
config = ConfigParser(interpolation=ExtendedInterpolation())
config.getlist = types.MethodType(get_list, config)
config.read(conf_file)
parser = argparse.ArgumentParser(prog='dnsmock',
formatter_class=WideHelpFormatter)
parser.add_argument("--config",
help="configuration file location",
dest="config",
default="/etc/dnsmock/dnsmock.conf")
parser.add_argument("--exthelp",
help="extended help",
dest="exthelp",
action="store_true")
