def run_cleaning(dry_run=False, underscore_flag=False):
"""method designed for running program"""
is_config_exists = os.path.exists(ConfigHandler.config_file)
if is_config_exists:
default_directory = ConfigHandler.get("default_directory")
else:
default_directory = "/home/" + getpass.getuser() + "/Downloads"
while True:
directory = input(
f"Default directory for cleaner is {default_directory}. \n"
f"If it's not, please input correct directory: ")
if directory != "" and os.path.isdir(directory):
break
elif directory == "" and os.path.isdir(default_directory):
directory = default_directory
break
else:
print("Provided path is not a directory.")
if directory != default_directory:
ConfigHandler.add(default_directory=directory)
cleaner = ChosenFolderHandler(directory,
dry_run=dry_run,
underscore_flag=underscore_flag)
cleaner.organize()
cleaner.clean()
def __init__(self, config=my_config.CONFIG_PATH, mytype='json', path=''):
if not any(mytype == _ for _ in self._support_types.keys()):
raise IOError('{0} type not in support_types {1}'.format(
mytype, self._support_types))
if not path:
config_handler = ConfigHandler(config)
path = config_handler.get_chain_config('DB', 'db_path')
self._type_db = self._support_types[mytype](path)
self._onchain_handler = ProvedDBOnChainHandler(config)
def undeploy(config_path=CONFIG_PATH):
''' Actually, smart contract cannot undeploy, but I need an function to remove unused intermediate file'''
config_handler = ConfigHandler(config_path)
contract_names = config_handler.get_chain_config('Deploy',
'target_contract_name')
for contract_name in contract_names.split(','):
contract_path = os.path.join(
config_handler.get_chain_config('Output', 'file_path'),
'{0}.json'.format(contract_name))
os.unlink(contract_path)
def __init__(self):
"""Setup... thats about it."""
self.auth_ = {}
# Settings/Menus/Errors
global _setup_errors
self.configObj = ConfigHandler()
self.configObj.setup_errors = _setup_errors
self.config = self.configObj.setup()
return None
def main():
feeds_file_input = os.path.abspath('config/feed_list.json')
db_config_file = os.path.abspath('config/db_config.json')
config_handler = ConfigHandler(feeds_file_input, db_config_file)
uri, db_name, feeds_name_collection, feeds_collection, image_collection = config_handler.get_db_config(
)
db_context = DbContext(uri, db_name, feeds_name_collection,
feeds_collection, image_collection)
Logger.log('reading {0} ...'.format(feeds_file_input))
feed_list, feed_list_jsondata = config_handler.load_feed_list()
Logger.log('collecting from {0} feeds'.format(len(feed_list)))
Logger.log('inserting the feed list into the database...')
for feed in feed_list_jsondata:
db_context.feeds_name_collection.update_one({'url': feed['url']},
{'$set': feed},
upsert=True)
images_path_file = 'config/image_collection.json'
images_path = config_handler.load_image_collection_path(images_path_file)
scraper = Scraper(feed_list, images_path)
entries = scraper.get_entries()
# get the metadata in interest and the images
with Pool() as pool:
metadata = pool.map(scraper.get_metadata, entries)
Logger.log('inserting metadata into the database...')
for feed_data in metadata:
db_context.feeds_collection.update_one({'link': feed_data['link']},
{'$set': feed_data},
upsert=True)
#db_context.feeds_collection.update_many(metadata, {'$set': metadata}, upsert=True)
metadata_number = db_context.feeds_collection.find({}).count()
Logger.log('{0} metadata inserted'.format(metadata_number))
Logger.log('creating indexes...')
# compound index
db_context.feeds_collection.create_index([('title', pymongo.TEXT),
('summary', pymongo.TEXT)],
default_language='english',
name='title_summary_index')
db_context.feeds_collection.create_index([("image_path", pymongo.ASCENDING)
])
Logger.log('downloading the images...')
scraper.download_images(metadata, images_path)
Logger.log('inserting image collection path into the database...')
full_img_path = scraper.download_dir
data = {'path': full_img_path}
db_context.image_collection.update_one(data, {'$set': data}, upsert=True)
Logger.log('all done.\n')
def __init__(self, contract_name, config_path=my_config.CONFIG_PATH):
self._config_handler = ConfigHandler(config_path)
self._check_contract_name(contract_name)
file_ipc = self._config_handler.get_chain_config(
'Ethereum', 'file_ipc')
self._w3 = Web3(Web3.IPCProvider(file_ipc))
contract_info = self._get_contract_info(contract_name)
contract_abi = contract_info['abi']
contract_address = contract_info['address']
self._contract_inst = self._w3.eth.contract(contract_address,
abi=contract_abi)
def run(self):
from config_handler import ConfigHandler
(HOST, PORT, CRYPT) = ConfigHandler().get_all("HTTPServer") # pylint: disable=unbalanced-tuple-unpacking
data = {
'id': 12,
'value': [10],
'type': 'line',
'active_points': 20,
'label': ['Random HTTP Number'],
'legend': ['random'],
'name': 'HTTP Example',
'borderColor': ['#3e95cd'],
'backgroundColor': ['#3e95cd'],
'api_crypt':CRYPT
}
"""
video_data= {
'id': 12,
'value': 0,
'type': 'video_stream',
'name': 'Video Stream HTTP Example',
'api_crypt':CRYPT
}
"""
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
API_ENDPOINT = "http://"+HOST+":"+PORT
while True:
# sending post request and saving response as response object
requests.post(url=API_ENDPOINT, data=json.dumps(data), headers=headers)
time.sleep(2)
def configure_files(force=False, config_file=CONFIG):
"""Configure server files."""
config = ConfigHandler(config_file)
configuration_times = {}
total_time = time.time()
for data_type, data_values in config.items():
start_time = time.time()
data_handler = DataFile.get_data_handler(data_type=data_type,
config=data_values,
server_config=config.server)
data_handler.configure(force)
configuration_times[data_type] = "%ss" % format(
time.time() - start_time, ".2f")
configuration_times["Total"] = "%ss" % format(time.time() - total_time,
".2f")
flog.info("Time to configure files:\n%s" % configuration_times)
return True
def run(self):
from config_handler import ConfigHandler
(HOST, PORT, CRYPT) = ConfigHandler().get_all("HTTPServer") # pylint: disable=unbalanced-tuple-unpacking
server_address = (str(HOST), int(PORT))
httpd = HTTPServer(server_address, partial(S, CRYPT))
try:
httpd.serve_forever()
except KeyboardInterrupt:
pass
httpd.server_close()
def test_config_handler_full():
ch = ConfigHandler("configs/full.yml")
full = deepcopy(baseline)
full["time_step"] = 2
full["limit_step"] = 2500000
full["interface_max_rate"] = 5000000
full["flowstat_window_size"] = 5
assert full == ch.config
def __init__(self, **kwargs):
self.config = ConfigHandler("config")
self.main_db = asyncio.get_event_loop().run_until_complete(
DatabaseHandler.create_instance())
self.up_time_start_time = get_current_time()
super(Bot, self).__init__(command_prefix=self.prefix_callable,
help_command=None,
description=self.config["bot_description"],
case_insensitive=True,
**kwargs)
class Security(commands.Cog):
def __init__(self, bot):
self.bot = bot
self.session = aiohttp.ClientSession()
self.banned_words = ConfigHandler("banned_words.json")
@commands.Cog.listener()
async def on_message(self, message):
ctx = message.channel
if message.guild is None or message.author == message.guild.me:
return
elif message.guild.id != tortoise_guild_id:
# Functionality only available in Tortoise guild
return
# Check for invites
# For this to work bot needs to have manage_guild permission (so he can get guild invites)
if "https:" in message.content or "http:" in message.content:
# Find any url
base_url = re.findall(r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+", message.content)
for invite in base_url:
# Get the endpoint of that url (for discord invite url shorteners)
async with self.session.get(invite) as response:
invite = str(response.url)
if "discordapp.com/invite/" in invite or "discord.gg/" in invite:
if not await Security.check_if_invite_is_our_guild(invite, message.guild):
await ctx.send(f"{message.author.mention} You are not allowed to post other guild invites!")
await message.delete()
# TODO give him warning points etc / send to deterrence channel
# Check for baned words
for word in message.content.lower().split():
for key in self.banned_words.loaded:
if word in self.banned_words.get_key(key):
await ctx.send(f"Curse word detected from category {key}!")
await message.delete()
# TODO: give him warning points or smth
@staticmethod
async def check_if_invite_is_our_guild(full_link, guild):
guild_invites = await guild.invites()
for invite in guild_invites:
# discord.gg/code resolves to https://discordapp.com/invite/code after using session.get(invite)
if Security._get_invite_link_code(invite.url) == Security._get_invite_link_code(full_link):
return True
return False
@staticmethod
def _get_invite_link_code(string: str):
return string.split("/")[-1]
def configure_server_rules(config_file=CONFIG,
with_filtering=True,
with_services=True):
"""Configure server rules."""
config = ConfigHandler(config_file)
try:
tc_manager = TrafficControl(config)
tc_manager.configure(with_filtering=with_filtering,
with_services=with_services)
tc_manager.show_rules()
return True
except AssertionError as ex:
flog.error(ex)
return False
class ContractHandler():
def __init__(self, contract_name, config_path=my_config.CONFIG_PATH):
self._config_handler = ConfigHandler(config_path)
self._check_contract_name(contract_name)
file_ipc = self._config_handler.get_chain_config(
'Ethereum', 'file_ipc')
self._w3 = Web3(Web3.IPCProvider(file_ipc))
contract_info = self._get_contract_info(contract_name)
contract_abi = contract_info['abi']
contract_address = contract_info['address']
self._contract_inst = self._w3.eth.contract(contract_address,
abi=contract_abi)
def _check_contract_name(self, check_name):
contract_names = self._config_handler.get_chain_config(
'Deploy', 'target_contract_name')
if check_name not in contract_names.split(','):
raise IOError('Cannot find {0} in config {1}'.format(
check_name, contract_names))
def get_w3(self):
return self._w3
def get_contract(self):
return self._contract_inst
def _get_contract_info(self, contract_name):
file_path = os.path.join(
self._config_handler.get_chain_config('Output', 'file_path'),
'{0}.json'.format(contract_name))
file_path = os.path.abspath(file_path)
with open(file_path) as f:
contract_info = json.load(f)
return contract_info
def run(self):
from config_handler import ConfigHandler
(HOST, PORT) = ConfigHandler().get_all("SocketServer") # pylint: disable=unbalanced-tuple-unpacking
#HOST = '127.0.0.1' # Standard loopback interface address (localhost)
#PORT = 65432 # Port to listen on (non-privileged ports are > 1023)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind((str(HOST), int(PORT)))
s.listen()
try:
while True:
conn, addr = s.accept()
print('Connected by', addr)
Thread(target=self.handle_connection,
args=(conn, )).start()
except Exception as e:
print(e)
def run(self):
from config_handler import ConfigHandler
(HOST, PORT) = ConfigHandler().get_all("SocketServer") # pylint: disable=unbalanced-tuple-unpacking
#HOST = '127.0.0.1' # Standard loopback interface address (localhost)
#PORT = 65432 # can change this if you want
# Create a TCP/IP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Bind the socket to the port
server_address = (str(HOST), int(PORT))
sock.connect(server_address)
while True:
sock.sendall(str.encode(self.message))
time.sleep(2)
def before_all(context):
config = ConfigHandler('app.ini')
app_host = config.get_config_option('bottle', 'host')
app_port = config.get_config_option('bottle', 'port')
context.app_uri = 'http://%s:%s/serve' % (app_host, app_port)
context.neo4j_conf = {
'server': config.get_config_option('neo4j', 'server'),
'user': config.get_config_option('neo4j', 'user'),
'password': config.get_config_option('neo4j', 'password'),
}
context.neo4j_adapter = Neo4jAdapter(context.neo4j_conf)
nlp_host = config.get_config_option('nlp', 'host')
nlp_port = config.get_config_option('nlp', 'port')
context.nlp_uri = 'http://%s:%s/?properties={"annotators":"sentiment","outputFormat":"json"}' % (
nlp_host, nlp_port)
def load_config(self, config_file):
self.config = ConfigHandler(config_file, default_config, option_types)
return self # allow chaining
for k, v in topics_conf.items()
] + ['"%s.%s_constraints": "%s"' % (TOPIC_PREFIX, prefix, statement)])
text_file.write(SINK_CONFIG %
(topic_list, host_conf['host'], host_conf['username'],
host_conf['password'], topics))
print('%s topics created.' % len(topics_conf))
if __name__ == '__main__':
if len(sys.argv) < 4:
print('python extractor.py <config> <source_dir> <target_dir>\n')
exit(1)
source_dir = sys.argv[2]
target_dir = sys.argv[3]
config_handler = ConfigHandler(sys.argv[1])
prefix = config_handler.get_config_option('info', 'prefix')
sources = config_handler.get_eval_option('extraction', 'sources')
for source in sources:
tsv_files = source['tsv_files']
for tsv_file in tsv_files:
file_name = '%s/%s' % (source_dir, tsv_file['file_name'])
lines = load_lines(file_name)
ent_conf = tsv_file['entities']
for entity_name, entity_info in ent_conf.items():
file_name = '%s/%s.tsv' % (target_dir, entity_name)
rows = create_rows(lines, entity_info)
count = write_rows(rows, file_name, entity_info)
from logging.config import fileConfig
from os import getpid, kill
from queue import Queue
import signal
from threading import Thread, Event, get_ident
from time import sleep
from bottle import HTTPError, run, route, request, response
from config_handler import ConfigHandler
from neo4j_adapter import Neo4jAdapter
fileConfig('logging.ini')
logger = logging.getLogger('appLogger')
config = ConfigHandler('app.ini')
bottle_conf = {
'host': config.get_config_option('bottle', 'host'),
'port': int(config.get_config_option('bottle', 'port')),
'server': config.get_config_option('bottle', 'server'),
'threads': int(config.get_config_option('bottle', 'threads')),
}
neo4j_conf = {
'server': config.get_config_option('neo4j', 'server'),
'user': config.get_config_option('neo4j', 'user'),
'password': config.get_config_option('neo4j', 'password'),
}
in_queue = Queue()
out_queue_dict = dict()
def check_options_standings(leagues, history):
if not leagues:
raise IncorrectParametersException("Please specify a league. "
"Example --standings --league=EN1")
if history:
raise IncorrectParametersException(
"--history and --days is not supported for --standings. "
"Use --matches to use these parameters")
for league in leagues:
if league.endswith("C") and league not in ["WC", "EC"]:
raise IncorrectParametersException(
f"Standings for {league} not supported")
ch = ConfigHandler()
def get_possible_leagues():
params = get_params(ch.get('auth', 'api_token'),
ch.get('profile', 'timezone'))
rh = RequestHandler(params, LEAGUES_DATA, None, ch)
leagues = rh.get_leagues()
return [convert.league_id_to_league_abbreviation(x['id']) for x in leagues]
@click.command()
@click.option('--api_token',
default=ch.load_config_file,
help="API key to use.")
@click.option('--timezone',
def __init__(self, bot):
self.bot = bot
self.session = aiohttp.ClientSession()
self.banned_words = ConfigHandler("banned_words.json")
import csv
import pandas as pd
from flask import Flask, render_template, request, redirect, url_for,\
make_response, jsonify
import predictor
from werkzeug.contrib.cache import SimpleCache
from config_handler import ConfigHandler
from pathlib import Path
from datastore import Datastore, PickledDatastore
config = ConfigHandler()
if config.has('redis') and config.get('redis'):
cache = Datastore()
config = ConfigHandler(PickledDatastore(cache))
else:
cache = SimpleCache()
config = ConfigHandler(cache)
app = Flask(__name__)
data = {}
if config.has('input', 'ieee'):
data['ieee'] = pd.read_csv(config.get('input', 'ieee'))
if config.has('input', 'acm'):
acm_data = pd.read_csv(config.get('input', 'acm'))
acm_data['url'] = acm_data['id'].apply(
"https://dl.acm.org/citation.cfm?id={}&preflayout=flat#abstract".format
)
acm_data['title'] = acm_data['title'].fillna(acm_data['booktitle'])
def get_db_path(config):
config_handler = ConfigHandler(config)
return config_handler.get_chain_config('DB', 'db_path')
kivy.require("2.0.0")
from kivy.config import Config
Config.set("graphics", "width", "480")
Config.set("graphics", "height", "320")
from audio_player import AudioPlayer
from config_handler import ConfigHandler
from constants import CONFIG_ENABLE_DARK_MODE_KEY
from kivymd.app import MDApp
from muezzin_carousel import MuezzinCarousel
audio_player = AudioPlayer()
config_handler = ConfigHandler()
enable_dark_mode = config_handler.get_setting(CONFIG_ENABLE_DARK_MODE_KEY)
class MuezzinApp(MDApp):
"""
Main app for Muezzing
"""
def build(self):
"""
Sets the icon and theme parameters of Kivy, and returns an instance of the MuezzinCarousel
:return: an instance of MuezzinCarousel
"""
self.icon = "res/mosque_green.png"
self.theme_cls.primary_palette = "Green"
ONE_DAY = 24 * 60 * 60
CONFIG_PATH = os.path.expanduser('~/.MozITP/')
# get command
if len(sys.argv) < 2:
exit(0)
else:
logger.debug('argv: {}'.format(sys.argv))
command = sys.argv[1]
logger.debug('Command: {}'.format(command))
hash_obj = hashlib.sha256()
hash_obj.update(command)
hash_code = hash_obj.hexdigest()
logger.debug('Hash file: ' + os.path.join(CONFIG_PATH, hash_code))
ch = ConfigHandler(CONFIG_PATH)
if ch.is_config_exist(hash_code):
now = time.time()
config_mtime = ch.get_config_mtime(hash_code)
diff_time = now - config_mtime
logger.info('Now: {}, Latest: {}, Diff: {}'.format(now, config_mtime, diff_time))
if diff_time > ONE_DAY:
logger.info('Run: {}'.format(command))
os.system(command)
logger.info('origin m-time {}'.format(config_mtime))
config_mtime = ch.touch_config(hash_code)
logger.info('new m-time {}'.format(config_mtime))
else:
logger.info('Skip: {}'.format(command))
else:
logger.info('Run: {}'.format(command))
def test_config_handler_yaml_syntax_error():
with pytest.raises(ParserError):
ConfigHandler("configs/syntax_error.yml")
import sys
from config_handler import ConfigHandler
if __name__ == '__main__':
config_handler = ConfigHandler(sys.argv[1])
option_dict = config_handler.get_eval_option(sys.argv[2], sys.argv[3])
print(option_dict[sys.argv[4]])
def main():
args = docopt(usage)
config = ConfigHandler(args)
if args["download"]:
Downloader(config)
def start_flask_application():
from config_handler import ConfigHandler
[HOST, PORT] = ConfigHandler().get_all("Flask") # pylint: disable=unbalanced-tuple-unpacking
socketio.run(app, host=HOST, port=PORT) # SocketIOServer
app.run(host=HOST, port=PORT) # Other Server
__maintainer__ = "Nghia Doan"
__email__ = "*****@*****.**"
__status__ = "Development"
from itertools import groupby
from typing import List
from fastapi import FastAPI
from pydantic import BaseModel
from requests import Session
from config_handler import ConfigHandler
####################
# Reading configuration from given file in relative path
config = ConfigHandler('conf/extractor.ini')
tika_url = config.get_config_option('tika', 'url')
file_dir = config.get_config_option('tika', 'dir')
####################
# Define the document model that the webapp receives from submission:
# It is a json format:
# [
# "u": the file name of the document, the webapp retains and returns it
# ]
class Item(BaseModel):
u: str
####################
