def s3_open(fname):
if fname is None or len(fname) == 0:
raise Exception('no credfile specfied')
cp = CP()
if fname in cp.read(fname):
access = cp.get('Credentials', 'aws_access_key_id')
secret = cp.get('Credentials', 'aws_secret_access_key')
return boto.connect_s3(aws_access_key_id=access,
aws_secret_access_key=secret)
else:
raise Exception('failed to load credentials')
def read_config(config_file='/usr/lib/nagios/plugins/ceph/rgw_test.conf'):
config = CP()
config.read(config_file)
cfg = {}
cfg['host'] = config.get('connection', 'host')
cfg['port'] = config.getint('connection', 'port')
cfg['ssl'] = config.getboolean('connection', 'ssl')
cfg['block'] = config.getint('test', 'urandom_block_size')
cfg['bytes'] = config.getint('test', 'total_bytes')
cfg['bucket'] = config.get('test', 'bucket')
cfg['uid'] = config.get('test', 'uid')
cfg['nagios_host'] = config.get('send_nsca', 'nagios_host')
return cfg
def add_library(self, name, library):
from fusesoc.provider import get_provider
if not hasattr(self, '_path'):
raise RuntimeError(
"No FuseSoC config file found - can't add library")
section_name = 'library.' + name
config = CP()
config.read(self._path)
if not section_name in config.sections():
config.add_section(section_name)
# This is not user-controlled at all so an assert is OK
assert ('location' in library or 'sync-uri' in library)
# sync-uri is absent for local libraries
if 'sync-uri' in library:
config.set(section_name, 'sync-uri', library['sync-uri'])
if 'sync-type' in library:
config.set(section_name, 'sync-type', library['sync-type'])
else:
library['sync-type'] = 'git'
if 'auto-sync' in library:
if library['auto-sync']:
config.set(section_name, 'auto-sync', 'true')
else:
config.set(section_name, 'auto-sync', 'false')
else:
library['auto-sync'] = True
if 'location' in library:
config.set(section_name, 'location', library['location'])
else:
library['location'] = os.path.join(self.library_root, name)
self.libraries[name] = library
try:
provider = get_provider(library['sync-type'])
except ImportError as e:
raise RuntimeError("Invalid sync-type '{}'".format(
library['sync-type']))
provider.init_library(library)
with open(self._path, 'w') as conf_file:
config.write(conf_file)
def _load_config_file(cfg_file):
"""Load a folder config file and return it as JSON."""
cfg_file = cfg_file or os.path.join(os.path.dirname(__file__), 'folder.cfg')
assert os.path.isfile(cfg_file), 'Failed to find config file at: %s' % cfg_file
parser = CP()
parser.read(cfg_file)
config = {}
for section in parser.sections():
config[section] = {}
for option in parser.options(section):
config[section][option] = \
parser.get(section, option).split('#')[0].strip()
return config, cfg_file.replace('\\', '/')
def main():
conf = CP()
conf.read('vk_settings.ini')
login, password = conf.get('Settings',
'Login'), conf.get('Settings', 'Password')
global vk
vk = vk_api.VkApi(login, password)
try:
vk.authorization()
except vk_api.AuthorizationError as error_msg:
print(error_msg)
return
#tools = vk_api.VkTools(vk)
print "Searching ", AGE_FROM
# what shall we look for?
data = {
'fields': 'screen_name,schools,sex,last_seen',
'city': 10,
'age_from': int(AGE_FROM),
'age_to': int(AGE_TO),
'count': 1000,
'sex': SEX,
'v': '5.41',
}
for chunk in CHUNKS:
# use execute to get data, because 'users.search' has restictions on 1000 ids
# params: chunk - list of dict's with school ids, data - json search
users = execute(chunk, data)
# use vk_api method to get data
# users = tools.get_all('users.search', 100, data)
# making dump. just in case smth will be wrong
with open('dump.txt', 'a') as fh:
pickle.dump(users, fh)
fh.close()
print 'Is there an error? or we\'ve retrieved %r users' % len(users)
# if we got something let's put it into db
if len(users) > 0:
for user in users:
put_into_db(user)
print 'Sleeping now...'
time.sleep(180)
# we've finished parsing, time to close db
conn.close()
def add_library(self, library):
from fusesoc.provider import get_provider
if not hasattr(self, "_path"):
raise RuntimeError("No FuseSoC config file found - can't add library")
section_name = "library." + library.name
config = CP()
config.read(self._path)
if section_name in config.sections():
logger.warning(
"Not adding library. {} already exists in configuration file".format(
library.name
)
)
return
config.add_section(section_name)
config.set(section_name, "location", library.location)
if library.sync_type:
config.set(section_name, "sync-uri", library.sync_uri)
config.set(section_name, "sync-type", library.sync_type)
_auto_sync = "true" if library.auto_sync else "false"
config.set(section_name, "auto-sync", _auto_sync)
try:
provider = get_provider(library.sync_type)
except ImportError as e:
raise RuntimeError("Invalid sync-type '{}'".format(library["sync-type"]))
provider.init_library(library)
with open(self._path, "w") as conf_file:
config.write(conf_file)
def add_library(self, library):
from fusesoc.provider import get_provider
if not hasattr(self, '_path'):
raise RuntimeError(
"No FuseSoC config file found - can't add library")
section_name = 'library.' + library.name
config = CP()
config.read(self._path)
if section_name in config.sections():
logger.warn(
"Not adding library. {} already exists in configuration file".
format(library.name))
return
config.add_section(section_name)
config.set(section_name, 'location', library.location)
if library.sync_type:
config.set(section_name, 'sync-uri', library.sync_uri)
config.set(section_name, 'sync-type', library.sync_type)
_auto_sync = 'true' if library.auto_sync else 'false'
config.set(section_name, 'auto-sync', _auto_sync)
try:
provider = get_provider(library.sync_type)
except ImportError as e:
raise RuntimeError("Invalid sync-type '{}'".format(
library['sync-type']))
provider.init_library(library)
with open(self._path, 'w') as conf_file:
config.write(conf_file)
def fallback_read():
try:
read_from_remote(fallback_filename,fallback_filename+'-rtn')
if fallback_only:
#atexit.register(report, hostname, nagios_service_name, NAG_OK, "OK: XRootD check successful (fallback only mode)", config)
print('atexit.register(report, hostname, nagios_service_name, NAG_OK, "OK: XRootD check successful (fallback only mode)", config)')
else:
#atexit.register(report, hostname, nagios_service_name, NAG_WARNING, "WARNING: Failed to write remote file, fallback read successful", config)
print('atexit.register(report, hostname, nagios_service_name, NAG_WARNING, "WARNING: Failed to write remote file, fallback read successful", config)')
except:
pass
sys.exit()
if __name__ == '__main__':
config = CP()
config.read(config_file)
hostname = socket.gethostname().split('.')[0]
nagios_service_name = 'cjp - Ceph XRootD functional test'
tempfsize = config.getint('tempfile','size') # tmpfsize=67108864, 64MB
tempfblksize = config.getint('tempfile','blocksize') # tmpfblksize=4194304, 4MB
fallback_filename = config.get('fallback','persistent_file_name') #TODO remove fallback / find out how works
fallback_only = config.getboolean('fallback','fallback_only')
#call grid_proxy_cmd = ['grid-proxy-init', '-cert', usercert, '-key', userkey, '-valid', validity]
#then apparently immediately delete the created tmpfile
set_environment(config)
def __init__(self, path=None, file=None):
self.build_root = None
self.cache_root = None
cores_root = []
systems_root = []
self.library_root = None
self.libraries = []
config = CP()
if file is None:
if path is None:
xdg_config_home = os.environ.get("XDG_CONFIG_HOME") or os.path.join(
os.path.expanduser("~"), ".config"
)
config_files = [
"/etc/fusesoc/fusesoc.conf",
os.path.join(xdg_config_home, "fusesoc", "fusesoc.conf"),
"fusesoc.conf",
]
else:
logger.debug("Using config file '{}'".format(path))
if not os.path.isfile(path):
with open(path, "a"):
pass
config_files = [path]
logger.debug("Looking for config files from " + ":".join(config_files))
files_read = config.read(config_files)
logger.debug("Found config files in " + ":".join(files_read))
if files_read:
self._path = files_read[-1]
else:
logger.debug("Using supplied config file")
if sys.version[0] == "2":
config.readfp(file)
else:
config.read_file(file)
file.seek(0)
self._path = file.name
for item in ["build_root", "cache_root", "systems_root", "library_root"]:
try:
setattr(self, item, os.path.expanduser(config.get("main", item)))
if item == "systems_root":
systems_root = [os.path.expanduser(config.get("main", item))]
logger.warning(
"The systems_root option in fusesoc.conf is deprecated. Please migrate to libraries instead"
)
except configparser.NoOptionError:
pass
except configparser.NoSectionError:
pass
try:
cores_root = config.get("main", "cores_root").split()
logger.warning(
"The cores_root option in fusesoc.conf is deprecated. Please migrate to libraries instead"
)
except configparser.NoOptionError:
pass
except configparser.NoSectionError:
pass
# Set fallback values
if self.build_root is None:
self.build_root = os.path.abspath("build")
if self.cache_root is None:
xdg_cache_home = os.environ.get("XDG_CACHE_HOME") or os.path.join(
os.path.expanduser("~"), ".cache"
)
self.cache_root = os.path.join(xdg_cache_home, "fusesoc")
os.makedirs(self.cache_root, exist_ok=True)
if not cores_root and os.path.exists("cores"):
cores_root = [os.path.abspath("cores")]
if (not systems_root) and os.path.exists("systems"):
systems_root = [os.path.abspath("systems")]
if self.library_root is None:
xdg_data_home = os.environ.get("XDG_DATA_HOME") or os.path.join(
os.path.expanduser("~"), ".local/share"
)
self.library_root = os.path.join(xdg_data_home, "fusesoc")
# Parse library sections
libraries = []
library_sections = [x for x in config.sections() if x.startswith("library")]
for section in library_sections:
name = section.partition(".")[2]
try:
location = config.get(section, "location")
except configparser.NoOptionError:
location = os.path.join(self.library_root, name)
try:
auto_sync = config.getboolean(section, "auto-sync")
except configparser.NoOptionError:
auto_sync = True
except ValueError as e:
_s = "Error parsing auto-sync '{}'. Ignoring library '{}'"
logger.warning(_s.format(str(e), name))
continue
try:
sync_uri = config.get(section, "sync-uri")
except configparser.NoOptionError:
# sync-uri is absent for local libraries
sync_uri = None
try:
sync_type = config.get(section, "sync-type")
except configparser.NoOptionError:
# sync-uri is absent for local libraries
sync_type = None
libraries.append(Library(name, location, sync_type, sync_uri, auto_sync))
# Get the environment variable for further cores
env_cores_root = []
if os.getenv("FUSESOC_CORES"):
env_cores_root = os.getenv("FUSESOC_CORES").split(":")
env_cores_root.reverse()
for root in cores_root + systems_root + env_cores_root:
self.libraries.append(Library(root, root))
self.libraries += libraries
logger.debug("cache_root=" + self.cache_root)
logger.debug("library_root=" + self.library_root)
def __init__(self, path=None, file=None):
self.build_root = None
self.cache_root = None
self.cores_root = []
self.systems_root = None
self.library_root = None
self.libraries = OrderedDict()
config = CP()
if file is None:
if path is None:
xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \
os.path.join(os.path.expanduser('~'), '.config')
config_files = [
'/etc/fusesoc/fusesoc.conf',
os.path.join(xdg_config_home, 'fusesoc', 'fusesoc.conf'),
'fusesoc.conf'
]
else:
logger.debug("Using config file '{}'".format(path))
if not os.path.isfile(path):
with open(path, 'a'):
pass
config_files = [path]
logger.debug('Looking for config files from ' +
':'.join(config_files))
files_read = config.read(config_files)
logger.debug('Found config files in ' + ':'.join(files_read))
if files_read:
self._path = files_read[-1]
else:
logger.debug('Using supplied config file')
if sys.version[0] == '2':
config.readfp(file)
else:
config.read_file(file)
file.seek(0)
self._path = file.name
for item in [
'build_root', 'cache_root', 'systems_root', 'library_root'
]:
try:
setattr(self, item,
os.path.expanduser(config.get('main', item)))
except configparser.NoOptionError:
pass
except configparser.NoSectionError:
pass
item = 'cores_root'
try:
setattr(self, item, config.get('main', item).split())
except configparser.NoOptionError:
pass
except configparser.NoSectionError:
pass
#Set fallback values
if self.build_root is None:
self.build_root = os.path.abspath('build')
if self.cache_root is None:
xdg_cache_home = os.environ.get('XDG_CACHE_HOME') or \
os.path.join(os.path.expanduser('~'), '.cache')
self.cache_root = os.path.join(xdg_cache_home, 'fusesoc')
if not os.path.exists(self.cache_root):
os.makedirs(self.cache_root)
if not self.cores_root and os.path.exists('cores'):
self.cores_root = [os.path.abspath('cores')]
if self.systems_root is None and os.path.exists('systems'):
self.systems_root = os.path.abspath('systems')
if self.library_root is None:
xdg_data_home = os.environ.get('XDG_DATA_HOME') or \
os.path.join(os.path.expanduser('~'), '.local/share')
self.library_root = os.path.join(xdg_data_home, 'fusesoc')
# Parse library sections
library_sections = [
x for x in config.sections() if x.startswith('library')
]
for section in library_sections:
library = section.partition('.')[2]
try:
location = config.get(section, 'location')
except configparser.NoOptionError:
location = os.path.join(self.library_root, library)
try:
auto_sync = config.getboolean(section, 'auto-sync')
except configparser.NoOptionError:
auto_sync = True
except ValueError as e:
_s = "Error parsing auto-sync '{}'. Ignoring library '{}'"
logger.warn(_s.format(str(e), library))
continue
try:
sync_uri = config.get(section, 'sync-uri')
except configparser.NoOptionError:
# sync-uri is absent for local libraries
sync_uri = None
try:
sync_type = config.get(section, 'sync-type')
except configparser.NoOptionError:
# sync-uri is absent for local libraries
sync_type = None
self.libraries[library] = {
'location': location,
'auto-sync': auto_sync,
'sync-uri': sync_uri,
'sync-type': sync_type
}
logger.debug('cache_root=' + self.cache_root)
logger.debug('cores_root=' + ':'.join(self.cores_root))
logger.debug('systems_root=' +
self.systems_root if self.systems_root else "Not defined")
logger.debug('library_root=' + self.library_root)
def __init__(self, path=None, file=None):
self.build_root = None
self.cache_root = None
cores_root = []
systems_root = []
self.library_root = None
self.libraries = []
config = CP()
if file is None:
if path is None:
xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \
os.path.join(os.path.expanduser('~'), '.config')
config_files = [
'/etc/fusesoc/fusesoc.conf',
os.path.join(xdg_config_home, 'fusesoc', 'fusesoc.conf'),
'fusesoc.conf'
]
else:
logger.debug("Using config file '{}'".format(path))
if not os.path.isfile(path):
with open(path, 'a'):
pass
config_files = [path]
logger.debug('Looking for config files from ' +
':'.join(config_files))
files_read = config.read(config_files)
logger.debug('Found config files in ' + ':'.join(files_read))
if files_read:
self._path = files_read[-1]
else:
logger.debug('Using supplied config file')
if sys.version[0] == '2':
config.readfp(file)
else:
config.read_file(file)
file.seek(0)
self._path = file.name
for item in [
'build_root', 'cache_root', 'systems_root', 'library_root'
]:
try:
setattr(self, item,
os.path.expanduser(config.get('main', item)))
if item == 'systems_root':
systems_root = [
os.path.expanduser(config.get('main', item))
]
logger.warn(
'The systems_root option in fusesoc.conf is deprecated. Please migrate to libraries instead'
)
except configparser.NoOptionError:
pass
except configparser.NoSectionError:
pass
try:
cores_root = config.get('main', 'cores_root').split()
logger.warn(
'The cores_root option in fusesoc.conf is deprecated. Please migrate to libraries instead'
)
except configparser.NoOptionError:
pass
except configparser.NoSectionError:
pass
#Set fallback values
if self.build_root is None:
self.build_root = os.path.abspath('build')
if self.cache_root is None:
xdg_cache_home = os.environ.get('XDG_CACHE_HOME') or \
os.path.join(os.path.expanduser('~'), '.cache')
self.cache_root = os.path.join(xdg_cache_home, 'fusesoc')
if not os.path.exists(self.cache_root):
os.makedirs(self.cache_root)
if not cores_root and os.path.exists('cores'):
cores_root = [os.path.abspath('cores')]
if (not systems_root) and os.path.exists('systems'):
systems_root = [os.path.abspath('systems')]
if self.library_root is None:
xdg_data_home = os.environ.get('XDG_DATA_HOME') or \
os.path.join(os.path.expanduser('~'), '.local/share')
self.library_root = os.path.join(xdg_data_home, 'fusesoc')
# Parse library sections
libraries = []
library_sections = [
x for x in config.sections() if x.startswith('library')
]
for section in library_sections:
name = section.partition('.')[2]
try:
location = config.get(section, 'location')
except configparser.NoOptionError:
location = os.path.join(self.library_root, name)
try:
auto_sync = config.getboolean(section, 'auto-sync')
except configparser.NoOptionError:
auto_sync = True
except ValueError as e:
_s = "Error parsing auto-sync '{}'. Ignoring library '{}'"
logger.warn(_s.format(str(e), name))
continue
try:
sync_uri = config.get(section, 'sync-uri')
except configparser.NoOptionError:
# sync-uri is absent for local libraries
sync_uri = None
try:
sync_type = config.get(section, 'sync-type')
except configparser.NoOptionError:
# sync-uri is absent for local libraries
sync_type = None
libraries.append(
Library(name, location, sync_type, sync_uri, auto_sync))
# Get the environment variable for further cores
env_cores_root = []
if os.getenv("FUSESOC_CORES"):
env_cores_root = os.getenv("FUSESOC_CORES").split(":")
env_cores_root.reverse()
for root in cores_root + systems_root + env_cores_root:
self.libraries.append(Library(root, root))
self.libraries += libraries
logger.debug('cache_root=' + self.cache_root)
logger.debug('library_root=' + self.library_root)
"photo104995591_397714630", "photo104995591_397714637",
"photo104995591_397714642", "photo104995591_397714648",
"photo104995591_397714652", "photo104995591_397714657",
"photo104995591_397714663", "photo104995591_397714666",
"photo104995591_397714671", "photo104995591_397714676",
"photo104995591_397714684", "photo104995591_397714687",
"photo104995591_397714697", "photo104995591_397714703",
"photo104995591_397714708", "photo104995591_397714711",
"photo104995591_397714718", "photo104995591_397714721",
"photo104995591_397714724", "photo104995591_397714728"
]
# get prepared...
driver = wd.Chrome()
driver.get('http://vk.com/')
conf = CP()
conf.read('vk_settings.ini')
conn = sqlite3.connect('vk_schoolboys.sqlite')
DB = conn.cursor()
# authorization. no sanity check (((
driver.find_element_by_name('email').send_keys(conf.get('Settings', 'Login'))
driver.find_element_by_name('pass').send_keys(conf.get('Settings', 'Password'))
driver.find_element_by_id('quick_login_button').click()
# aux functions
def rnd_time():
return random.random() * 8