#!/usr/bin/python
from ConfigParser import RawConfigParser
from email.mime.base import MIMEBase
import email
import email.message
import re
import GnuPG
import smtplib
import sys
import syslog
# Read configuration from /etc/gpg-mailgate.conf
_cfg = RawConfigParser()
_cfg.read('/etc/gpg-mailgate.conf')
cfg = dict()
for sect in _cfg.sections():
cfg[sect] = dict()
for (name, value) in _cfg.items(sect):
cfg[sect][name] = value
def log(msg):
if cfg.has_key('logging') and cfg['logging'].has_key('file'):
if cfg['logging']['file'] == "syslog":
syslog.syslog(syslog.LOG_INFO | syslog.LOG_MAIL, msg)
else:
logfile = open(cfg['logging']['file'], 'a')
logfile.write(msg + "\n")
logfile.close()
def read_key_from_config():
config = RawConfigParser()
config.read(os.path.join(os.path.dirname(sys.argv[0]), 'domLink.cfg'))
return config.get('API_KEYS', 'whoxy')
"--emevoucher_key",
dest="emevoucher_key",
help="The certificate to use for signing the eme voucher")
parser.add_option(
"--emevoucher_chain",
dest="emevoucher_chain",
help="Certificate chain to include in EME voucher signatures")
parser.add_option("-v",
action="store_const",
dest="loglevel",
const=logging.DEBUG)
options, args = parser.parse_args()
if options.configfile:
config = RawConfigParser()
config.read(options.configfile)
for option, value in config.items('signscript'):
if option == "signcode_timestamp":
value = config.getboolean('signscript', option)
options.ensure_value(option, value)
# Reset to default if this wasn't set in the config file
if options.signcode_timestamp is None:
options.signcode_timestamp = True
logging.basicConfig(level=options.loglevel,
format="%(asctime)s - %(message)s")
if len(args) != 4:
parser.error("Incorrect number of arguments")
def expand_macro(self, formatter, name, content, args=None):
if content is not None:
content = content.strip()
if not args and not content:
raw_actions = self.config.options('ticket-workflow')
else:
is_macro = args is None
if is_macro:
kwargs = parse_args(content)[1]
file = kwargs.get('file')
else:
file = args.get('file')
if not file and not content:
raise ProcessorError("Invalid argument(s).")
if file:
print(file)
text = RepositoryManager(self.env).read_file_by_path(file)
if text is None:
raise ProcessorError(
tag_("The file %(file)s does not exist.",
file=tag.code(file)))
elif is_macro:
text = '\n'.join(line.lstrip() for line in content.split(';'))
else:
text = content
if '[ticket-workflow]' not in text:
text = '[ticket-workflow]\n' + text
parser = RawConfigParser()
try:
parser.readfp(io.StringIO(text))
except ParsingError as e:
if is_macro:
raise MacroError(exception_to_unicode(e))
else:
raise ProcessorError(exception_to_unicode(e))
raw_actions = list(parser.items('ticket-workflow'))
actions = parse_workflow_config(raw_actions)
states = list(
{state for action in actions.itervalues()
for state in action['oldstates']} |
{action['newstate'] for action in actions.itervalues()})
action_labels = [attrs['label'] for attrs in actions.values()]
action_names = list(actions)
edges = []
for name, action in actions.items():
new_index = states.index(action['newstate'])
name_index = action_names.index(name)
for old_state in action['oldstates']:
old_index = states.index(old_state)
edges.append((old_index, new_index, name_index))
args = args or {}
width = args.get('width', 800)
height = args.get('height', 600)
graph = {'nodes': states, 'actions': action_labels, 'edges': edges,
'width': width, 'height': height}
graph_id = '%012x' % id(graph)
req = formatter.req
add_script(req, 'common/js/excanvas.js', ie_if='IE')
add_script(req, 'common/js/workflow_graph.js')
add_script_data(req, {'graph_%s' % graph_id: graph})
return tag(
tag.div('', class_='trac-workflow-graph trac-noscript',
id='trac-workflow-graph-%s' % graph_id,
style="display:inline-block;width:%spx;height:%spx" %
(width, height)),
tag.noscript(
tag.div(_("Enable JavaScript to display the workflow graph."),
class_='system-message')))
def load_config(environ):
"""Load configuration options
Options are read from a config file. The config file location is
controlled by the PythonOption ConfigFile in the httpd config.
Backwards compatibility:
- if ConfigFile is not set, opts are loaded from http config
- if ConfigFile is set, then the http config must not provide Koji options
- In a future version we will load the default hub config regardless
- all PythonOptions (except ConfigFile) are now deprecated and support for them
will disappear in a future version of Koji
"""
logger = logging.getLogger("koji")
#get our config file(s)
cf = environ.get('koji.hub.ConfigFile', '/etc/koji-hub/hub.conf')
cfdir = environ.get('koji.hub.ConfigDir', '/etc/koji-hub/hub.conf.d')
if cfdir:
configs = koji.config_directory_contents(cfdir)
else:
configs = []
if cf and os.path.isfile(cf):
configs.append(cf)
if configs:
config = RawConfigParser()
config.read(configs)
else:
config = None
cfgmap = [
#option, type, default
['DBName', 'string', None],
['DBUser', 'string', None],
['DBHost', 'string', None],
['DBhost', 'string', None], # alias for backwards compatibility
['DBPass', 'string', None],
['KojiDir', 'string', None],
['AuthPrincipal', 'string', None],
['AuthKeytab', 'string', None],
['ProxyPrincipals', 'string', ''],
['HostPrincipalFormat', 'string', None],
['DNUsernameComponent', 'string', 'CN'],
['ProxyDNs', 'string', ''],
['CheckClientIP', 'boolean', True],
['LoginCreatesUser', 'boolean', True],
['KojiWebURL', 'string', 'http://localhost.localdomain/koji'],
['EmailDomain', 'string', None],
['NotifyOnSuccess', 'boolean', True],
['DisableNotifications', 'boolean', False],
['Plugins', 'string', ''],
['PluginPath', 'string', '/usr/lib/koji-hub-plugins'],
['KojiDebug', 'boolean', False],
['KojiTraceback', 'string', None],
['VerbosePolicy', 'boolean', False],
['EnableFunctionDebug', 'boolean', False],
['LogLevel', 'string', 'WARNING'],
[
'LogFormat', 'string',
'%(asctime)s [%(levelname)s] m=%(method)s u=%(user_name)s p=%(process)s r=%(remoteaddr)s %(name)s: %(message)s'
],
['MissingPolicyOk', 'boolean', True],
['EnableMaven', 'boolean', False],
['EnableWin', 'boolean', False],
['EnableImageMigration', 'boolean', False],
['RLIMIT_AS', 'string', None],
['RLIMIT_CORE', 'string', None],
['RLIMIT_CPU', 'string', None],
['RLIMIT_DATA', 'string', None],
['RLIMIT_FSIZE', 'string', None],
['RLIMIT_MEMLOCK', 'string', None],
['RLIMIT_NOFILE', 'string', None],
['RLIMIT_NPROC', 'string', None],
['RLIMIT_OFILE', 'string', None],
['RLIMIT_RSS', 'string', None],
['RLIMIT_STACK', 'string', None],
['MemoryWarnThreshold', 'integer', 5000],
['MaxRequestLength', 'integer', 4194304],
['LockOut', 'boolean', False],
['ServerOffline', 'boolean', False],
['OfflineMessage', 'string', None],
]
opts = {}
for name, dtype, default in cfgmap:
key = ('hub', name)
if config and config.has_option(*key):
if dtype == 'integer':
opts[name] = config.getint(*key)
elif dtype == 'boolean':
opts[name] = config.getboolean(*key)
else:
opts[name] = config.get(*key)
continue
opts[name] = default
if opts['DBHost'] is None:
opts['DBHost'] = opts['DBhost']
# load policies
# (only from config file)
if config and config.has_section('policy'):
#for the moment, we simply transfer the policy conf to opts
opts['policy'] = dict(config.items('policy'))
else:
opts['policy'] = {}
for pname, text in _default_policies.iteritems():
opts['policy'].setdefault(pname, text)
# use configured KojiDir
if opts.get('KojiDir') is not None:
koji.BASEDIR = opts['KojiDir']
koji.pathinfo.topdir = opts['KojiDir']
return opts
#!/usr/bin/env python
import sys
import os
import logging
from gntp import __version__
from gntp.notifier import GrowlNotifier
from optparse import OptionParser, OptionGroup
from ConfigParser import RawConfigParser
DEFAULT_CONFIG = os.path.expanduser('~/.gntp')
config = RawConfigParser({
'hostname': 'localhost',
'password': None,
'port': 23053,
})
config.read([DEFAULT_CONFIG])
if not config.has_section('gntp'):
logging.info('Error reading ~/.gntp config file')
config.add_section('gntp')
class ClientParser(OptionParser):
def __init__(self):
OptionParser.__init__(self, version="%%prog %s" % __version__)
group = OptionGroup(self, "Network Options")
group.add_option(
"-H",
"--host",
def __init__(self):
"""Parse configuration and CLI options."""
global config_file
# look for an alternative configuration file
alt_config_file = False
# used to show errors before we actually start parsing stuff
parser = OptionParser()
for arg in sys.argv:
if arg == '--config':
try:
alt_config_file = sys.argv[sys.argv.index(arg) + 1]
config_file = alt_config_file
except IndexError:
pass
elif arg.startswith('--config='):
_, alt_config_file = arg.split('=', 1)
if alt_config_file == '':
parser.error("--config option requires an argument")
config_file = alt_config_file
config = RawConfigParser({
'auth-url':
None,
'host-key-file':
None,
'bind-address':
"127.0.0.1",
'port':
8022,
'memcache':
None,
'max-children':
"40",
'auth-timeout':
"30",
'negotiation-timeout':
"30",
'log-file':
None,
'syslog':
'no',
'verbose':
'no',
'scp-support':
'yes',
'pid-file':
None,
'uid':
None,
'gid':
None,
'split-large-files':
"1073741824",
'hide-part-dir':
"no",
# keystone auth 2.0 support
'keystone-auth':
False,
'keystone-region-name':
None,
'keystone-tenant-separator':
default_ks_tenant_separator,
'keystone-service-type':
default_ks_service_type,
'keystone-endpoint-type':
default_ks_endpoint_type,
})
if not config.read(config_file) and alt_config_file:
# the default conf file is optional
parser.error("failed to read %s" % config_file)
if not config.has_section('sftpcloudfs'):
config.add_section('sftpcloudfs')
parser = OptionParser(version="%prog " + version,
description="This is a SFTP interface to OpenStack " + \
"Object Storage (Swift).",
epilog="Contact and support at: %s" % project_url)
parser.add_option("-a",
"--auth-url",
dest="authurl",
default=config.get('sftpcloudfs', 'auth-url'),
help="Authentication URL")
parser.add_option("-k",
"--host-key-file",
dest="host_key",
default=config.get('sftpcloudfs', 'host-key-file'),
help="Host RSA key used by the server")
parser.add_option("-b",
"--bind-address",
dest="bind_address",
default=config.get('sftpcloudfs', 'bind-address'),
help="Address to bind (default: 127.0.0.1)")
parser.add_option("-p",
"--port",
dest="port",
type="int",
default=config.get('sftpcloudfs', 'port'),
help="Port to bind (default: 8022)")
memcache = config.get('sftpcloudfs', 'memcache')
if memcache:
memcache = [x.strip() for x in memcache.split(',')]
parser.add_option(
'--memcache',
type="str",
dest="memcache",
action="append",
default=memcache,
help="Memcache server(s) to be used for cache (ip:port)")
parser.add_option("-l",
"--log-file",
dest="log_file",
default=config.get('sftpcloudfs', 'log-file'),
help="Log into provided file")
parser.add_option(
"-f",
"--foreground",
dest="foreground",
action="store_true",
default=False,
help="Run in the foreground (don't detach from terminal)")
parser.add_option(
"--disable-scp",
dest="no_scp",
action="store_true",
default=not config.getboolean('sftpcloudfs', 'scp-support'),
help="Disable SCP support (default: enabled)")
parser.add_option(
"--syslog",
dest="syslog",
action="store_true",
default=config.getboolean('sftpcloudfs', 'syslog'),
help="Enable logging to system logger (daemon facility)")
parser.add_option("-v",
"--verbose",
dest="verbose",
action="store_true",
default=config.getboolean('sftpcloudfs', 'verbose'),
help="Show detailed information on logging")
parser.add_option('--pid-file',
type="str",
dest="pid_file",
default=config.get('sftpcloudfs', 'pid-file'),
help="Pid file location when in daemon mode")
parser.add_option(
'--uid',
type="int",
dest="uid",
default=config.get('sftpcloudfs', 'uid'),
help="UID to drop the privileges to when in daemon mode")
parser.add_option(
'--gid',
type="int",
dest="gid",
default=config.get('sftpcloudfs', 'gid'),
help="GID to drop the privileges to when in daemon mode")
parser.add_option(
'--keystone-auth',
action="store_true",
dest="keystone",
default=config.get('sftpcloudfs', 'keystone-auth'),
help="Use auth 2.0 (Keystone, requires keystoneclient)")
parser.add_option('--keystone-region-name',
type="str",
dest="region_name",
default=config.get('sftpcloudfs',
'keystone-region-name'),
help="Region name to be used in auth 2.0")
parser.add_option('--keystone-tenant-separator',
type="str",
dest="tenant_separator",
default=config.get('sftpcloudfs', 'keystone-tenant-separator'),
help="Character used to separate tenant_name/username in auth 2.0, " + \
"default: TENANT%sUSERNAME" % default_ks_tenant_separator)
parser.add_option(
'--keystone-service-type',
type="str",
dest="service_type",
default=config.get('sftpcloudfs', 'keystone-service-type'),
help="Service type to be used in auth 2.0, default: %s" %
default_ks_service_type)
parser.add_option(
'--keystone-endpoint-type',
type="str",
dest="endpoint_type",
default=config.get('sftpcloudfs', 'keystone-endpoint-type'),
help="Endpoint type to be used in auth 2.0, default: %s" %
default_ks_endpoint_type)
parser.add_option('--config',
type="str",
dest="config",
default=config_file,
help="Use an alternative configuration file")
(options, args) = parser.parse_args()
# required parameters
if not options.authurl:
parser.error("No auth-url provided")
if not options.host_key:
parser.error("No host-key-file provided")
try:
self.host_key = paramiko.RSAKey(filename=options.host_key)
except (IOError, paramiko.SSHException), e:
parser.error("host-key-file: %s" % e)
def _read_config(self):
config = RawConfigParser()
with codecs.open(self.confpath, 'r', encoding='utf-8') as fd:
config.readfp(fd)
return config
def save(self):
config = RawConfigParser()
for name, version in self.versions.iteritems():
config.set(DEFAULTSECT, name, version)
with open(os.path.join(self.path, self.VERSIONS_LIST), 'wb') as fp:
config.write(fp)
def clear(self):
self._config = RawConfigParser()
def test_call_signature(self, m_time_delta, m_req):
m_res = m_req.return_value
m_json = m_res.json.return_value
m_time_delta.return_value = 42
api = Client(ENDPOINT, APPLICATION_KEY, APPLICATION_SECRET, CONSUMER_KEY)
# nominal
m_res.status_code = 200
self.assertEqual(m_json, api.call(FAKE_METHOD, FAKE_PATH, None, True))
m_time_delta.assert_called_once_with()
m_req.assert_called_once_with(
FAKE_METHOD, BASE_URL+'/unit/test',
headers={
'X-Ovh-Consumer': CONSUMER_KEY,
'X-Ovh-Application': APPLICATION_KEY,
'X-Ovh-Signature': '$1$16ae5ba8c63841b1951575be905867991d5f49dc',
'X-Ovh-Timestamp': '1404395931',
}, data='', timeout=TIMEOUT
)
m_time_delta.reset_mock()
m_req.reset_mock()
# data, nominal
data = OrderedDict([('some', 'random'), ('data', 42)])
m_res.status_code = 200
self.assertEqual(m_json, api.call(FAKE_METHOD, FAKE_PATH, data, True))
m_time_delta.assert_called_once_with()
m_req.assert_called_once_with(
FAKE_METHOD, BASE_URL+'/unit/test',
headers={
'X-Ovh-Consumer': CONSUMER_KEY,
'X-Ovh-Application': APPLICATION_KEY,
'Content-type': 'application/json',
'X-Ovh-Timestamp': '1404395931',
'X-Ovh-Signature': '$1$9acb1ac0120006d16261a635aed788e83ab172d2',
}, data=json.dumps(data), timeout=TIMEOUT
)
m_time_delta.reset_mock()
m_req.reset_mock()
# Overwrite configuration to avoid interfering with any local config
from ovh.client import config
try:
from ConfigParser import RawConfigParser
except ImportError:
# Python 3
from configparser import RawConfigParser
self._orig_config = config.config
config.config = RawConfigParser()
# errors
try:
api = Client(ENDPOINT, APPLICATION_KEY, None, CONSUMER_KEY)
self.assertRaises(InvalidKey, api.call, FAKE_METHOD, FAKE_PATH, None, True)
api = Client(ENDPOINT, APPLICATION_KEY, APPLICATION_SECRET, None)
self.assertRaises(InvalidKey, api.call, FAKE_METHOD, FAKE_PATH, None, True)
finally:
# Restore configuration
config.config = self._orig_config
def load(self, fp):
self._config = RawConfigParser()
try:
self._config.readfp(fp)
except ConfigParser.Error as e:
raise InvalidConfigurationError(str(e))
def __init__(self):
self._config = RawConfigParser()
# A convenient place for other code to store a file name.
self.target_file = None
def setUp(self):
self.testParser = RawConfigParser()
for section in self._parser.sections():
self.testParser.add_section(section)
for item in self._parser.items(section):
self.testParser.set(section, item[0], item[1])
def __init__(self, log=None):
super(MyMQTT, self).__init__()
self.LogFileName = "/var/log/genmqtt.log"
if log != None:
self.log = log
else:
# log errors in this module to a file
self.log = mylog.SetupLogger("client", self.LogFileName)
# cleanup
# test
self.console = mylog.SetupLogger("mymqtt_console",
log_file="",
stream=True)
self.Username = None
self.Password = None
self.Address = None
self.Topic = None
self.MQTTAddress = "127.0.0.1"
self.MonitorAddress = "127.0.0.1"
self.Port = 1883
self.Topic = "generator"
self.TopicRoot = None
self.BlackList = None
self.PollTime = 2
self.FlushInterval = float(
'inf') # default to inifite flush interval (e.g., never)
self.Debug = False
try:
# read config file
config = RawConfigParser()
# config parser reads from current directory, when running form a cron tab this is
# not defined so we specify the full path
config.read('/etc/genmqtt.conf')
CONFIG_SECTION = "genmqtt"
if config.has_option(CONFIG_SECTION, 'username'):
self.Username = config.get(CONFIG_SECTION, 'username')
if config.has_option(CONFIG_SECTION, 'password'):
self.Password = config.get(CONFIG_SECTION, 'password')
self.MQTTAddress = config.get(CONFIG_SECTION, 'mqtt_address')
if config.has_option(CONFIG_SECTION, 'monitor_address'):
self.MonitorAddress = config.get(CONFIG_SECTION,
'monitor_address')
if config.has_option(CONFIG_SECTION, 'mqtt_port'):
self.MQTTPort = config.getint(CONFIG_SECTION, 'mqtt_port')
if config.has_option(CONFIG_SECTION, 'poll_interval'):
self.PollTime = config.getfloat(CONFIG_SECTION,
'poll_interval')
if config.has_option(CONFIG_SECTION, 'root_topic'):
self.TopicRoot = config.get(CONFIG_SECTION, 'root_topic')
if config.has_option(CONFIG_SECTION, 'blacklist'):
BlackList = config.get(CONFIG_SECTION, 'blacklist')
self.BlackList = BlackList.strip().split(",")
if config.has_option(CONFIG_SECTION, 'debug'):
self.Debug = config.getboolean(CONFIG_SECTION, 'debug')
if config.has_option(CONFIG_SECTION, 'flush_interval'):
self.FlushInterval = config.getfloat(CONFIG_SECTION,
'flush_interval')
except Exception as e1:
log.error("Error reading /etc/genmqtt.conf: " + str(e1))
console.error("Error reading /etc/genmqtt.conf: " + str(e1))
sys.exit(1)
try:
self.MQTTclient = mqtt.Client(client_id="genmon")
if self.Username != None and len(
self.Username) and self.Password != None:
self.MQTTclient.username_pw_set(self.Username,
password=self.Password)
self.MQTTclient.on_connect = self.on_connect
self.MQTTclient.on_message = self.on_message
self.MQTTclient.connect(self.MQTTAddress, self.Port, 60)
self.Push = MyGenPush(host=self.MonitorAddress,
log=self.log,
callback=self.PublishCallback,
polltime=self.PollTime,
blacklist=self.BlackList,
flush_interval=self.FlushInterval)
atexit.register(self.Close)
signal.signal(signal.SIGTERM, self.Close)
signal.signal(signal.SIGINT, self.Close)
self.MQTTclient.loop_start()
except Exception as e1:
self.LogErrorLine("Error in MyMQTT init: " + str(e1))
self.console.error("Error in MyMQTT init: " + str(e1))
sys.exit(1)
import tornado
from tornado import websocket
from os import path
import socket
import sys
import glob
from ConfigParser import RawConfigParser
config = RawConfigParser(allow_no_value=True)
config.read(path.join(path.dirname(sys.argv[0]), 'config.ini'))
clientdir = path.join(path.dirname(path.dirname(path.abspath(sys.argv[0]))),
'client')
class I2DXTopHandler(tornado.web.RequestHandler):
def get(self):
self.redirect('static/index.html')
def _glob(*largs):
return [
x[len(clientdir):] for x in glob.glob(path.join(clientdir, *largs))
]
class I2DXComponentsHandler(tornado.web.RequestHandler):
def get(self):
self.set_header('Content-Type', 'text/javascript')
for list in (_glob('layouts', '*.js'), _glob('skins', '*', 'skin.js')):
for file in list:
select
package app_package
from
ronghui_mart.app_info
where
(class_one is null or class_one in ('N', 'NULL', ''))
and (class_two is null or class_two in ('N', 'NULL', ''))
and (class_three is null or class_three in ('N', 'NULL', ''))
"""
apps = spark.sql(sql)
return apps
if __name__ == '__main__':
print('====> Initializing Spark APP')
localConf = RawConfigParser()
localConf.read('../config')
sparkConf = SparkConf()
for t in localConf.items('spark-config'):
sparkConf.set(t[0], t[1])
spark = SparkSession.builder \
.appName('RLab_Stats_Report___Prepare_App_Category_Data') \
.config(conf=sparkConf) \
.enableHiveSupport() \
.getOrCreate()
sc = spark.sparkContext
sc.setLogLevel('ERROR')
print('====> Parsing local arguments')
print('====> Start calculation')
#
# PortableApps.com Launcher Manual build configuration file, created by
# sphinx-quickstart on Wed Feb 17 13:51:33 2010.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
import sys, os
from ConfigParser import RawConfigParser
appinfo = RawConfigParser()
appinfo.read(
os.path.abspath(
os.path.dirname(__file__) + os.path.sep + '..' + os.path.sep + '..' +
os.path.sep + '..' + os.path.sep + 'App' + os.path.sep + 'AppInfo' +
os.path.sep + 'appinfo.ini'))
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.append(os.path.abspath('_ext'))
# -- General configuration -----------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
def main(argv):
(project, release, releaseDate) = ('ensembl', '', '')
configFile = 'void_rdf.cfg'
(skip_file, skip_qc, skip_push) = (False, False, False)
try:
opts, args = getopt.getopt(argv, "hc:p:r:d:", ['skip-file', 'skip-qc', 'skip-push'])
except getopt.GetoptError:
usage()
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
usage()
sys.exit()
elif opt == '-c':
configFile = arg
elif opt == '-p':
project = arg
elif opt == '-r':
release = int(arg)
elif opt == '-d':
releaseDate = datetime.datetime.strptime(arg, "%d-%m-%Y").date()
elif opt == '--skip-file':
skip_file = True
elif opt == '--skip-qc':
skip_qc = True
elif opt == '--skip-push':
skip_push = True
if not project or not release or not releaseDate:
usage()
sys.exit(2)
if project not in ('ensembl', 'ensemblgenomes'):
print("Error: project must be either 'ensembl' or 'ensemblgenomes'", file = sys.stderr)
sys.exit(2)
if not os.path.isfile(configFile):
print("Error: config file %s does not exist\n" % configFile, file = sys.stderr)
usage()
sys.exit(2)
voidFile = 'void.ttl'
zipped = False
if project == 'ensemblgenomes':
voidFile = 'ensemblgenomes_' + voidFile + '.gz'
zipped = True
voidRdf = None
if not skip_file:
### Retrieve species info (name, core/xref rdf paths)
# for each species with RDF in the project FTP area
print("Retrieving species RDF info from %s FTP site ..." % project)
speciesInfo = ProjectFTP(project).parseSpecies()
### Create Void RDF graph
print('Generating %s VOID RDF ...' % project)
voidRdf = VoidRDF(project, release, releaseDate, speciesInfo)
voidRdf.generate()
### Dump VOID RDF to file
print('Serialising to turtle file %s ...' % voidFile)
voidRdf.write(voidFile, zipped)
else:
print('Skipping VOID file generation')
if not skip_qc:
### QC of the VOID file
# raise AttributeError if it does not pass, do not handle
if skip_file:
print('Cannot QC VOID file with --skip-file option', file = sys.stderr)
sys.exit(2)
if not voidRdf:
print("Could not generate VOID RDF. Abort QC ...")
sys.exit(2)
print('Doing QC ...')
voidRdf.qc()
else:
print('Skipping VOID file QC')
if not skip_push:
### Push to the project specific branch of the RDF platform github repo
# read configuration to get necessary parameters, i.e. branch and token
if not os.path.isfile(voidFile):
print('Cannot push %s: no such file' % voidFile, file = sys.stderr)
sys.exit(2)
config = RawConfigParser()
config.read(configFile)
branch = config.get(project, 'branch')
user = config.get(project, 'user')
token = config.get(project, 'token')
if not branch:
raise AttributeError("Couldn\'t get github branch name for %s from config file" % project)
if not user:
raise AttributeError("Couldn\'t get github user name for %s from config file" % project)
if not token:
raise AttributeError("Couldn\'t get github valid access token for %s from config file" % project)
print('Pushing %s to EBI SPOT GitHub repo (branch: %s)' % (voidFile, branch))
push_to_repo_branch(voidFile, voidFile, branch, user, token)
print("Done.")
else:
print('Skipping pushing the file to EBI SPOT repo')
def package(source_metadata,
skip_sign=False,
sign_key=None,
source_only=False):
"""
Packages a given package.
"""
distribution, version_string, revision_date = source_metadata
build_opts = []
if skip_sign:
build_opts.append('-uc -us')
elif sign_key:
# TODO: if GPG key is password-protected, signing will fail because
# check_output is not interactive
build_opts.append('-k"%s"' % sign_key)
if source_only:
build_opts.append('-S')
filename = '{0}/../settings.cfg'.format(
os.path.dirname(os.path.abspath(__file__)))
settings = RawConfigParser()
settings.read(filename)
package_name = settings.get('packaging', 'package_name')
repo_path_code = SourceCollector.repo_path_code.format(
settings.get('packaging', 'working_dir'), package_name)
package_path = SourceCollector.package_path.format(
settings.get('packaging', 'working_dir'), package_name)
# Prepare
# /<pp>/debian
debian_folder = '{0}/debian'.format(package_path)
if os.path.exists(debian_folder):
shutil.rmtree(debian_folder)
# /<rp>/packaging/debian -> /<pp>/debian
shutil.copytree('{0}/packaging/debian'.format(repo_path_code),
debian_folder)
# Rename tgz
# /<pp>/<packagename>_1.2.3.tar.gz -> /<pp>/debian/<packagename>_1.2.3.orig.tar.gz
shutil.copyfile(
'{0}/{1}_{2}.tar.gz'.format(package_path, package_name,
version_string),
'{0}/{1}_{2}.orig.tar.gz'.format(debian_folder, package_name,
version_string))
# /<pp>/debian/<packagename>-1.2.3/...
SourceCollector.run(command='tar -xzf {0}_{1}.orig.tar.gz'.format(
package_name, version_string),
working_directory=debian_folder)
# Move the debian package metadata into the extracted source
# /<pp>/debian/debian -> /<pp>/debian/<packagename>-1.2.3/
SourceCollector.run(command='mv {0}/debian {0}/{1}-{2}/'.format(
debian_folder, package_name, version_string),
working_directory=package_path)
# Build changelog entry
with open(
'{0}/{1}-{2}/debian/changelog'.format(debian_folder,
package_name,
version_string),
'w') as changelog_file:
changelog_file.write("""{0} ({1}-1) {2}; urgency=low
* For changes, see individual changelogs
-- Packaging System <*****@*****.**> {3}
""".format(package_name, version_string, distribution,
revision_date.strftime('%a, %d %b %Y %H:%M:%S +0000')))
# Some more tweaks
SourceCollector.run(
command='chmod 770 {0}/{1}-{2}/debian/rules'.format(
debian_folder, package_name, version_string),
working_directory=package_path)
SourceCollector.run(
command="sed -i -e 's/__NEW_VERSION__/{0}/' *.*".format(
version_string),
working_directory='{0}/{1}-{2}/debian'.format(
debian_folder, package_name, version_string))
# Build the package
SourceCollector.run(
command='dpkg-buildpackage %s' % ' '.join(build_opts),
working_directory='{0}/{1}-{2}'.format(debian_folder, package_name,
version_string))
def load_config(self):
""" Read the config file for feeds to run """
now = time.localtime()
now_time = now.tm_hour * 60 + now.tm_min
config = RawConfigParser()
config.read(self.CONFIG_FILE)
for s in config.sections():
if not config.has_option(s, 'feed'):
print("feed '%s' missing 'feed' identifier" % (s))
continue
feed = self.feed_loader(config.get(s, 'feed'))
if not feed:
print("feed '%s' could not load module '%s'" %
(s, config.get(s, 'feed')))
continue
if not config.has_option(s, 'mode'):
print("feed '%s' missing 'mode' identifier" % (s))
continue
mode = config.get(s, 'mode').lower()
args = {}
for o in filter(lambda x: x.startswith('@'), config.options(s)):
args[o[1:]] = config.get(s, o)
feed_item = {'id': s, 'feed': feed, 'args': args, 'state': {}}
if mode == 'off':
pass
elif mode == 'start':
self.run_start.append(feed_item)
elif mode == 'stop':
self.run_stop.append(feed_item)
elif mode == 'hold':
self.run_hold.append(feed_item)
elif mode == 'tap':
self.run_tap.append(feed_item)
elif mode == 'interval':
if not config.has_option(s, 'interval'):
print("feed '%s' missing 'interval' value" % (s))
continue
try:
feed_item['interval'] = int(config.get(s, 'interval'))
except:
print("feed '%s' has invalid 'interval' value" % (s))
continue
feed_item['next'] = 0
self.run_interval.append(feed_item)
elif mode == 'at':
if not config.has_option(s, 'when'):
print("feed '%s' missing 'at' value" % (s))
continue
try:
t = time.strptime(config.get(s, 'when'), "%H:%M")
feed_item['when'] = t.tm_hour * 60 + t.tm_min
except:
print("feed '%s' has invalid 'at' value" % (s))
continue
if feed_item['when'] > now_time or self.RUN_SCHEDULED_AT_START:
feed_item['ran_today'] = False
else:
feed_item['ran_today'] = True
self.run_when.append(feed_item)
else:
print("feed '%s' has bad 'mode' value '%s'" % (s, mode))
continue
def __init__(self, filename):
self.cfg = RawConfigParser()
self.cfg.read(filename)
def load(self, context=None):
"""Attempt to load configuration from the current given path.
:param context: Context to be interpolated.
:type context: dict
:rtype: bool
:returns: Returns ``True`` if the path could be loaded. Also sets ``is_loaded``.
.. versionchanged:: 0.28.0-d
Added ``context`` parameter and support for interpolation in the config.
"""
if not path_exists(self.path):
self.is_loaded = False
self._error = "Configuration file does not exist: %s" % self.path
return False
# Load the config without interpolation.
if isinstance(context, dict):
ini = SafeConfigParser(defaults=context)
try:
ini.read(self.path)
except ParsingError as e:
self.is_loaded = False
self._error = e.message
return False
else:
ini = RawConfigParser()
try:
ini.read(self.path)
except ParsingError as e:
self.is_loaded = False
self._error = e.message
return False
# Iterate through the sections.
for section_name in ini.sections():
# Values will be passed as kwargs to the Section instance.
kwargs = {}
# Tags are handled specifically for all configurations.
try:
for key, value in ini.items(section_name):
if key == "tags":
kwargs['tags'] = value.split(",")
else:
kwargs[key] = value
except InterpolationMissingOptionError as e:
self.is_loaded = False
self._error = e.message
return False
# Load the section. This allows section initialization to be customized in child classes.
self._load_section(section_name, kwargs)
self.is_loaded = True
return True
def get_options():
"""
Read options from command line or from configuration file.
"""
default_database = rel('%s.db' % __script__)
names = ('database', 'password', 'username')
result = {}
# At first, read options from command line
args = sys.argv[1:]
description = 'Script to check who of your Facebook friends were removed '\
'from FB or unfriend you.'
parser = ArgumentParser(description=description)
add_argument(parser,
'-u',
'--username',
default=None,
dest='username',
metavar='FACEBOOK_USERNAME')
add_argument(parser,
'-p',
'--password',
default=None,
dest='password',
metavar='FACEBOOK_PASSWORD')
add_argument(parser,
'-d',
'--database',
default=None,
dest='database',
help='Path to database. By default: %s' % default_database)
try:
options, _ = parser.parse_args(args)
except TypeError:
options = parser.parse_args(args)
for name in names:
result.update({name: getattr(options, name, None)})
# Next, check for configuration file values
if not result['password'] or not result['username']:
config_file = rel('%s.conf' % __script__)
config_parser = RawConfigParser()
try:
config_parser.read(config_file)
except ConfigParserError:
pass
else:
values = config_parser.defaults()
for name in names:
if name == 'database' and result['database']:
continue
result.update({name: values.get(name)})
# Finally, get ability to input configuration
for name in ('username', 'password'):
if result[name]:
continue
func = getpass.getpass if name == 'password' else raw_input
result.update({name: func('Facebook %s: ' % name)})
if not result['database']:
result['database'] = default_database
return result.copy()
def cfg_to_args(path='setup.cfg'):
"""Compatibility helper to use setup.cfg in setup.py.
This functions uses an existing setup.cfg to generate a dictionary of
keywords that can be used by distutils.core.setup(**kwargs). It is used
by generate_setup_py.
*file* is the path to the setup.cfg file. If it doesn't exist,
PackagingFileError is raised.
"""
# XXX ** == needs testing
D1_D2_SETUP_ARGS = {
"name": ("metadata", ),
"version": ("metadata", ),
"author": ("metadata", ),
"author_email": ("metadata", ),
"maintainer": ("metadata", ),
"maintainer_email": ("metadata", ),
"url": ("metadata", "home_page"),
"description": ("metadata", "summary"),
"long_description": ("metadata", "description"),
"download-url": ("metadata", ),
"classifiers": ("metadata", "classifier"),
"platforms": ("metadata", "platform"), # **
"license": ("metadata", ),
"requires": ("metadata", "requires_dist"),
"provides": ("metadata", "provides_dist"), # **
"obsoletes": ("metadata", "obsoletes_dist"), # **
"package_dir": ("files", 'packages_root'),
"packages": ("files", ),
"scripts": ("files", ),
"py_modules": ("files", "modules"), # **
}
MULTI_FIELDS = ("classifiers", "platforms", "requires", "provides",
"obsoletes", "packages", "scripts", "py_modules")
def has_get_option(config, section, option):
if config.has_option(section, option):
return config.get(section, option)
elif config.has_option(section, option.replace('_', '-')):
return config.get(section, option.replace('_', '-'))
else:
return False
# The real code starts here
config = RawConfigParser()
f = codecs.open(path, encoding='utf-8')
try:
config.readfp(f)
finally:
f.close()
kwargs = {}
for arg in D1_D2_SETUP_ARGS:
if len(D1_D2_SETUP_ARGS[arg]) == 2:
# The distutils field name is different than distutils2's
section, option = D1_D2_SETUP_ARGS[arg]
else:
# The distutils field name is the same thant distutils2's
section = D1_D2_SETUP_ARGS[arg][0]
option = arg
in_cfg_value = has_get_option(config, section, option)
if not in_cfg_value:
# There is no such option in the setup.cfg
if arg == 'long_description':
filenames = has_get_option(config, section, 'description-file')
if filenames:
filenames = split_multiline(filenames)
in_cfg_value = []
for filename in filenames:
fp = codecs.open(filename, encoding='utf-8')
try:
in_cfg_value.append(fp.read())
finally:
fp.close()
in_cfg_value = '\n\n'.join(in_cfg_value)
else:
continue
if arg == 'package_dir' and in_cfg_value:
in_cfg_value = {'': in_cfg_value}
if arg in MULTI_FIELDS:
# support multiline options
in_cfg_value = split_multiline(in_cfg_value)
kwargs[arg] = in_cfg_value
return kwargs
def __nested__(func):
from ConfigParser import RawConfigParser
conf = RawConfigParser()
conf.read(conf_file)
items = dict(conf.items('database'))
return PgConnProxy(items, func)
def from_config(cls, filename=None, fp=None):
"""
ClusterConfiguration.from_config(filename=None, fp=None)
Read the configuration from the specified filename or file handle.
If neither are specified, /etc/slurm-ec2.conf is used.
"""
cp = RawConfigParser()
app_config_cp = RawConfigParser()
# Set up defaults here so we don't need try/except blocks around every
# cp.get() call below.
for key, value in cls.defaults.iteritems():
cp.set(DEFAULTSECT, key, value)
if filename is None and fp is None:
filename = "/etc/slurm-ec2.conf"
if fp is not None:
if filename is not None:
raise ValueError("Cannot specify both filename and fp")
cp.readfp(fp)
app_config_cp.readfp(fp)
else:
cp.read(filename)
app_config_cp.read(filename)
# Utility for splitting a string into a list if present; returns
# None if not present.
def parse_list(value):
if value is None:
return None
return value.split()
# Utility for converting a string into an int if present; returns
# None if not present
def parse_int(value):
if value is None:
return None
return int(value)
kw = {}
for key in cls.defaults.iterkeys():
value = cp.get(cls.master_config_section, key)
# Convert lists and integers
if key in cls.list_keys:
value = parse_list(value)
elif key in cls.int_keys:
value = parse_int(value)
kw[key] = value
if kw.get("vpc_id"):
# Parse the VPC section; get the list of subnets it contains.
vpc_id = kw['vpc_id']
subnet_ids = parse_list(cp.get(vpc_id, "subnet_ids"))
subnets = []
# Parse each subnet and create a Boto subnet object without
# querying the EC2 endpoint.
for subnet_id in subnet_ids:
cidr_block = cp.get(subnet_id, "cidr_block")
availability_zone = cp.get(subnet_id, "availability_zone")
subnet = Subnet()
subnet.id = subnet_id
subnet.vpc_id = vpc_id
subnet.cidr_block = cidr_block
subnet.available_ip_address_count = (
len(IPNetwork(cidr_block)) - 4)
subnet.availability_zone = availability_zone
subnets.append(subnet)
kw["_all_subnets"] = subnets
# We use the app_config_cp config parser here to avoid polluting
# application config with our defaults.
app_config = {}
for appname in app_config_cp.sections():
if (appname == "slurm-ec2" or appname.startswith("vpc-")
or appname.startswith("subnet-")):
continue
app_config[appname] = cp.items(appname)
kw["app_config"] = app_config
return cls(**kw)
def parse_mapping(fileobj, filename=None):
"""Parse an extraction method mapping from a file-like object.
>>> buf = StringIO('''
... [extractors]
... custom = mypackage.module:myfunc
...
... # Python source files
... [python: **.py]
...
... # Genshi templates
... [genshi: **/templates/**.html]
... include_attrs =
... [genshi: **/templates/**.txt]
... template_class = genshi.template:TextTemplate
... encoding = latin-1
...
... # Some custom extractor
... [custom: **/custom/*.*]
... ''')
>>> method_map, options_map = parse_mapping(buf)
>>> len(method_map)
4
>>> method_map[0]
('**.py', 'python')
>>> options_map['**.py']
{}
>>> method_map[1]
('**/templates/**.html', 'genshi')
>>> options_map['**/templates/**.html']['include_attrs']
''
>>> method_map[2]
('**/templates/**.txt', 'genshi')
>>> options_map['**/templates/**.txt']['template_class']
'genshi.template:TextTemplate'
>>> options_map['**/templates/**.txt']['encoding']
'latin-1'
>>> method_map[3]
('**/custom/*.*', 'mypackage.module:myfunc')
>>> options_map['**/custom/*.*']
{}
:param fileobj: a readable file-like object containing the configuration
text to parse
:see: `extract_from_directory`
"""
extractors = {}
method_map = []
options_map = {}
parser = RawConfigParser()
parser._sections = odict(parser._sections) # We need ordered sections
if PY2:
parser.readfp(fileobj, filename)
else:
parser.read_file(fileobj, filename)
for section in parser.sections():
if section == 'extractors':
extractors = dict(parser.items(section))
else:
method, pattern = [part.strip() for part in section.split(':', 1)]
method_map.append((pattern, method))
options_map[pattern] = dict(parser.items(section))
if extractors:
for idx, (pattern, method) in enumerate(method_map):
if method in extractors:
method = extractors[method]
method_map[idx] = (pattern, method)
return method_map, options_map
def __init__(self):
self.cfg_file = os.path.join(os.path.dirname(__file__), 'setup.conf')
self._config = RawConfigParser()
self._config.read(self.cfg_file)
For more information on this file, see
https://docs.djangoproject.com/en/1.10/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.10/ref/settings/
"""
import os
from ConfigParser import RawConfigParser
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CONFIG_FILE_PATH = '/home/artem/technotrack/web/conf/project.conf'
configs = RawConfigParser()
configs.read(CONFIG_FILE_PATH)
# print "CONFIGS:"
# for section in configs.sections():
# print " Section %s:" % section
# for item in configs.items(section):
# print " %s: %s" % (item[0], item[1])
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = configs.get('global', 'SECRET_KEY')
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
