def get_config(inFile):
P = configparser.RawConfigParser()
P.optionxform = str
P.read(inFile)
return P
import sys
import os
import time
import getpass
import logging
from sqlalchemy import *
import configparser as cp
import pandas as pd
# parameter
config_file = 'reeem_io_config.ini'
config_section = 'reeem'
log_file = 'reeem_adapter.log'
# sys.tracebacklimit = 0
cfg = cp.RawConfigParser()
def logger():
"""Configure logging in console and log file.
Returns
-------
rl : logger
Logging in console (ch) and file (fh).
"""
# set root logger (rl)
rl = logging.getLogger('REEEMLogger')
rl.setLevel(logging.INFO)
rl.propagate = False
import argparse
import configparser
import tcgen
import executer
import utils
parser = argparse.ArgumentParser()
config = configparser.RawConfigParser(allow_no_value=False)
parser.add_argument('-T',
metavar='--testcases',
type=str,
required=True,
help='file containg testcase syntax')
parser.add_argument('-I1',
metavar='--input1',
type=str,
required=False,
help='file containg code1')
parser.add_argument('-I2',
metavar='--input2',
type=str,
required=False,
help='file containg code2')
parser.add_argument('-C',
metavar='--config',
type=str,
default='configs/default.ini',
help='config file')
parser.add_argument('-N',
gr = "\033[1;32m"
cy = "\033[1;36m"
def banner():
print(f"""
{re}╔╦╗{cy}┌─┐┬ ┌─┐{re}╔═╗ ╔═╗{cy}┌─┐┬─┐┌─┐┌─┐┌─┐┬─┐
{re} ║ {cy}├┤ │ ├┤ {re}║ ╦ ╚═╗{cy}│ ├┬┘├─┤├─┘├┤ ├┬┘
{re} ╩ {cy}└─┘┴─┘└─┘{re}╚═╝ ╚═╝{cy}└─┘┴└─┴ ┴┴ └─┘┴└─
version : 3.1
youtube.com/theunknon
""")
cpass = configparser.RawConfigParser()
cpass.read('config.data')
try:
api_id = cpass['cred']['id']
api_hash = cpass['cred']['hash']
phone = cpass['cred']['phone']
client = TelegramClient(phone, api_id, api_hash)
except KeyError:
os.system('clear')
banner()
print(re + "[!] run python3 setup.py first !!\n")
sys.exit(1)
client.connect()
if not client.is_user_authorized():
def parse_venue_info(response):
"""Return only venue data from the given fetch_venue response"""
json_response = response.json()
venue = json_response.get('response', {}).get('venue')
return venue
def write_fetched_companies(companies):
"""Save a compressed CSV file with the given DF"""
companies.to_csv(OUTPUT_DATASET_PATH, compression='xz', index=False)
# API Keys
# You can create your own through https://pt.foursquare.com/developers/register
settings = configparser.RawConfigParser()
settings.read('config.ini')
CLIENT_ID = settings.get('Foursquare', 'ClientId')
CLIENT_SECRET = settings.get('Foursquare', 'ClientSecret')
# Foursquare API Version. This is in YYYYMMDD format.
VERSION = '20161021'
# Required params to make a request to Foursquare's API
DEFAULT_PARAMS = {
'client_id': CLIENT_ID,
'client_secret': CLIENT_SECRET,
'v': VERSION
}
# Dataset paths
REIMBURSEMENTS_DATASET_PATH = find_newest_file('reimbursements')
COMPANIES_DATASET_PATH = find_newest_file('companies')
def init_config(self, p=None):
print("Loading configuration properties...")
try:
config = configparser.RawConfigParser()
config.read('config.properties')
self.model_repo_host = ModelImageLib.read_config_option(config, 'Config', 'model.repo.host')
if self.model_repo_host is None or len(self.model_repo_host) < 1:
print("Please set Model Repository access URL in model.repo.host field inside config.properties file!")
return False
verbose_str = ModelImageLib.read_config_option(config, 'Config', 'verbose')
if verbose_str is not None and verbose_str == 'True':
self.verbose_on = True
else:
self.verbose_on = False
if p is not None:
self.provider = p
else:
self.provider = ModelImageLib.read_config_option(config, 'Config', 'provider.type')
self.kubernetes_context = ModelImageLib.read_config_option(config, self.provider, 'kubernetes.context')
registry = None
if self.provider == 'AWS':
# login amazon ecr
self.mm_docker_aws = CloudAWSLib(config, self.docker_client)
registry = self.mm_docker_aws.login()
elif self.provider == 'GCP':
# login Google Container Registry
self.mm_docker_gcp = CloudGCPLib(config, self.docker_client)
registry = self.mm_docker_gcp.login()
elif self.provider == 'Azure':
# login Azure Container Registry
self.mm_docker_azure = CloudAzureLib(config, self.docker_client)
registry = self.mm_docker_azure.login()
else:
registry = ModelImageLib.read_config_option(config, self.provider, 'base.repo')
if registry is None:
return False
self.convert_base_repo(registry)
# TODO only initialize k8s for certain actions
if self.kubernetes_context is not None and len(self.kubernetes_context) > 0:
print("Initializing kubernetes configuration...", self.kubernetes_context)
self.k8s = K8sLib(self.provider, self.kubernetes_context, self.verbose_on)
except:
print("Error loading configuration from config.properties file! Double-check Docker daemon or other environment.")
print(traceback.format_exc())
return False
print(' verbose:', self.verbose_on)
print(' model.repo.host:', self.model_repo_host)
print(' provider.type:', self.provider)
print(' base.repo:', self.base_repo)
print(' kubernetes.context:', self.kubernetes_context)
print("===================================")
return True
from ..resources import finder
backport_package = __name__.rsplit('.', 1)[0]
_finder = finder(backport_package)
_cfgfile = _finder.find('sysconfig.cfg')
assert _cfgfile, 'sysconfig.cfg exists'
with _cfgfile.as_stream() as s:
_SCHEMES.readfp(s)
if _PYTHON_BUILD:
for scheme in ('posix_prefix', 'posix_home'):
_SCHEMES.set(scheme, 'include', '{srcdir}/Include')
_SCHEMES.set(scheme, 'platinclude', '{projectbase}/.')
_cfg_read = True
_SCHEMES = configparser.RawConfigParser()
_VAR_REPL = re.compile(r'\{([^{]*?)\}')
def _expand_globals(config):
_ensure_cfg_read()
if config.has_section('globals'):
globals = config.items('globals')
else:
globals = tuple()
sections = config.sections()
for section in sections:
if section == 'globals':
continue
for option, value in globals:
def main():
parser = argparse.ArgumentParser('Parse configuration file')
parser.add_argument('--env_config', type=str, default='configs/env.config')
parser.add_argument('--policy', type=str, default='sarl')
parser.add_argument('--policy_config',
type=str,
default='configs/policy.config')
parser.add_argument('--train_config',
type=str,
default='configs/train.config') # changed config
parser.add_argument('--output_dir', type=str, default='data/output')
parser.add_argument('--weights', type=str)
parser.add_argument('--resume', default=False, action='store_true')
parser.add_argument('--gpu', default=False, action='store_true')
parser.add_argument('--debug', default=False, action='store_true')
args = parser.parse_args()
# configure paths
make_new_dir = True
if os.path.exists(args.output_dir):
key = input(
'Output directory already exists! Overwrite the folder? (y/n)')
if key == 'y' and not args.resume:
shutil.rmtree(args.output_dir)
else:
make_new_dir = False
args.env_config = os.path.join(args.output_dir,
os.path.basename(args.env_config))
args.policy_config = os.path.join(
args.output_dir, os.path.basename(args.policy_config))
args.train_config = os.path.join(
args.output_dir, os.path.basename(args.train_config))
if make_new_dir:
os.makedirs(args.output_dir)
shutil.copy(args.env_config, args.output_dir)
shutil.copy(args.policy_config, args.output_dir)
shutil.copy(args.train_config, args.output_dir)
log_file = os.path.join(args.output_dir, 'output.log')
il_weight_file = os.path.join(args.output_dir, 'il_model.pth')
rl_weight_file = os.path.join(args.output_dir, 'rl_model.pth')
# configure logging
mode = 'a' if args.resume else 'w'
file_handler = logging.FileHandler(log_file, mode=mode)
stdout_handler = logging.StreamHandler(sys.stdout)
level = logging.INFO if not args.debug else logging.DEBUG
logging.basicConfig(level=level,
handlers=[stdout_handler, file_handler],
format='%(asctime)s, %(levelname)s: %(message)s',
datefmt="%Y-%m-%d %H:%M:%S")
repo = git.Repo(search_parent_directories=True)
logging.info('Current git head hash code: %s'.format(
repo.head.object.hexsha))
device = torch.device(
"cuda:0" if torch.cuda.is_available() and args.gpu else "cpu")
logging.info('Using device: %s', device)
# configure policy
policy = policy_factory[args.policy]()
##--debug--###
# print('debug: train.py 68 args.polcy = ',args.policy)
# print('policy = ', policy)
# time.sleep(3)
if not policy.trainable:
parser.error('Policy has to be trainable')
if args.policy_config is None:
parser.error(
'Policy config has to be specified for a trainable network')
policy_config = configparser.RawConfigParser()
policy_config.read(args.policy_config)
policy.configure(policy_config)
policy.set_device(device)
# configure environment
env_config = configparser.RawConfigParser()
env_config.read(args.env_config)
env = gym.make('CrowdSim-v0')
env.configure(env_config)
robot = Robot(env_config, 'robot')
env.set_robot(robot)
# read training parameters
if args.train_config is None:
parser.error(
'Train config has to be specified for a trainable network')
train_config = configparser.RawConfigParser()
train_config.read(args.train_config)
rl_learning_rate = train_config.getfloat('train', 'rl_learning_rate')
train_batches = train_config.getint('train', 'train_batches')
train_episodes = train_config.getint('train', 'train_episodes')
sample_episodes = train_config.getint('train', 'sample_episodes')
target_update_interval = train_config.getint('train',
'target_update_interval')
evaluation_interval = train_config.getint('train', 'evaluation_interval')
capacity = train_config.getint('train', 'capacity')
epsilon_start = train_config.getfloat('train', 'epsilon_start')
epsilon_end = train_config.getfloat('train', 'epsilon_end')
epsilon_decay = train_config.getfloat('train', 'epsilon_decay')
checkpoint_interval = train_config.getint('train', 'checkpoint_interval')
# configure trainer and explorer
memory = ReplayMemory(capacity)
model = policy.get_model()
# print('----------------------debug--train.py109----------------')
# print('policy = ',policy)
# print('model = ',model)
batch_size = train_config.getint('trainer', 'batch_size') #batch_size =91
trainer = Trainer(model, memory, device, batch_size)
explorer = Explorer(env,
robot,
device,
memory,
policy.gamma,
target_policy=policy)
# imitation learning
if args.resume:
if not os.path.exists(rl_weight_file):
logging.error('RL weights does not exist')
model.load_state_dict(torch.load(rl_weight_file))
rl_weight_file = os.path.join(args.output_dir, 'resumed_rl_model.pth')
logging.info(
'Load reinforcement learning trained weights. Resume training')
elif os.path.exists(il_weight_file):
model.load_state_dict(torch.load(il_weight_file))
logging.info('Load imitation learning trained weights.')
else:
il_episodes = train_config.getint('imitation_learning', 'il_episodes')
il_policy = train_config.get('imitation_learning', 'il_policy')
il_epochs = train_config.getint('imitation_learning', 'il_epochs')
il_learning_rate = train_config.getfloat('imitation_learning',
'il_learning_rate')
trainer.set_learning_rate(il_learning_rate)
if robot.visible:
safety_space = 0
else:
safety_space = train_config.getfloat('imitation_learning',
'safety_space')
il_policy = policy_factory[il_policy]()
il_policy.multiagent_training = policy.multiagent_training
il_policy.safety_space = safety_space
robot.set_policy(il_policy)
file_name = strftime("%Y-%m-%d %H:%M", gmtime())
file_name = file_name + "_raw.txt"
print(file_name)
time.sleep((3))
f = open(file_name, "w+")
explorer.run_k_episodes(il_episodes,
'train',
file_name=file_name,
update_memory=True,
imitation_learning=True)
time.sleep(5)
f.close()
data_sort(file_name)
sys.exit(1)
trainer.optimize_epoch(il_epochs)
torch.save(model.state_dict(), il_weight_file)
logging.info('Finish imitation learning. Weights saved.')
logging.info('Experience set size: %d/%d', len(memory),
memory.capacity)
explorer.update_target_model(model)
# reinforcement learning
policy.set_env(env)
robot.set_policy(policy)
robot.print_info()
trainer.set_learning_rate(rl_learning_rate)
# fill the memory pool with some RL experience
if args.resume:
robot.policy.set_epsilon(epsilon_end)
explorer.run_k_episodes(100,
'train',
file_name=file_name,
update_memory=True,
episode=0)
logging.info('Experience set size: %d/%d', len(memory),
memory.capacity)
print('Il finished .............')
print('RL STARETED .............')
time.sleep(3)
## start training the model!
episode = 0
while episode < train_episodes: #train_episodes=10000
if args.resume:
epsilon = epsilon_end
else:
if episode < epsilon_decay:
epsilon = epsilon_start + (
epsilon_end - epsilon_start) / epsilon_decay * episode
else:
epsilon = epsilon_end
robot.policy.set_epsilon(epsilon)
# evaluate the model
if episode % evaluation_interval == 0:
#print('env.case_size[val] = ',env.case_size['val'])
time.sleep(3)
explorer.run_k_episodes(env.case_size['val'],
'val',
episode=episode)
# sample k episodes into memory and optimize over the generated memory
#print('sample_episodes = ',sample_episodes)
explorer.run_k_episodes(sample_episodes,
'train',
file_name=file_name,
update_memory=True,
episode=episode)
trainer.optimize_batch(train_batches)
episode += 1
#print('episode = ',episode)
if episode % target_update_interval == 0:
explorer.update_target_model(model)
if episode != 0 and episode % checkpoint_interval == 0:
torch.save(model.state_dict(), rl_weight_file)
# final test
explorer.run_k_episodes(env.case_size['test'], 'test', episode=episode)
import spotipy.oauth2
if sys.version_info.major >= 3:
import configparser
WRITE_MODE = 'w'
else:
import ConfigParser as configparser
WRITE_MOD = 'wb'
PREFS_FILE = os.environ['HOME'] + '/.config/spotify/barconfig.cfg'
REDIRECT_URI = 'http://localhost'
SCOPE = 'user-library-read user-library-modify '
# Parse all config under the "authorization" header
SECTION = 'authorization'
prefs = configparser.RawConfigParser()
prefs.readfp(open(PREFS_FILE))
# Check whether to use the cached value
def get_track_cache(track):
try:
cached = json.loads(prefs.get(SECTION, 'cache'))
if time.time() - cached['time'] < 20 and cached['track'] == track:
return cached['saved']
return None
except:
return None
def save_track_cache(track, val):
prefs.set(SECTION, 'cache', json.dumps({ 'time': time.time(), 'saved': val, 'track': track }))
with open(PREFS_FILE, WRITE_MODE) as f:
import os
import re
import configparser
import pafy
from pydub.exceptions import CouldntEncodeError
from pyytdl import PyYtDl
pyytdl = PyYtDl()
conf_params = configparser.RawConfigParser()
config_file_path = "config.ini"
conf_params.read(config_file_path)
download_list_location = os.path.join(
conf_params.get("config", r"download_list_location"))
output_directory = os.path.join(conf_params.get("config", r"output_directory"))
output_format = conf_params.get("config", "output_format").strip()
mtd = conf_params.get("config", r"meta_tagging_delimiter").strip()
if not os.path.isdir(output_directory):
os.makedirs(output_directory)
if not output_directory.endswith("\\"):
output_directory += "\\"
file = open(download_list_location, "r")
regex_mask = re.compile(r"" + re.escape(mtd) + ".*" + re.escape(mtd) + ".*")
for line in file:
is_metadata = False
if (regex_mask.search(line)):
def parse_args_and_config_file():
""" Read options from config file and CLI args
1. Reads hard coded DEFAULTS
2. Supersedes by values in config file
3. Supersedes by values from CLI args
"""
# Build preparser with only config-file and account
preparser = argparse.ArgumentParser(
# Turn off help in first parser because all options are not present
add_help=False)
preparser.add_argument('--account',
'-a',
metavar='STR',
help=('ledger account used as source'
' (default: {0})'.format(DEFAULTS.account)))
preparser.add_argument('--config-file',
'-c',
metavar='FILE',
help=('configuration file'
' (default search order: {0})'.format(
', '.join(FILE_DEFAULTS.config_file))))
# Parse args with preparser, and find config file
args, remaining_argv = preparser.parse_known_args()
args.config_file = find_first_file(args.config_file,
FILE_DEFAULTS.config_file)
# Initialize configparser with DEFAULTS, and then read config file
if args.config_file and ('-h' not in remaining_argv
and '--help' not in remaining_argv):
config = configparser.RawConfigParser(DEFAULTS)
config.read(args.config_file)
if not config.has_section(args.account):
print('Config file {0} does not contain section {1}'.format(
args.config_file, args.account),
file=sys.stderr)
sys.exit(1)
defaults = dict(config.items(args.account))
defaults['addons'] = {}
if config.has_section(args.account + '_addons'):
for item in config.items(args.account + '_addons'):
if item not in config.defaults().items():
defaults['addons']['addon_' + item[0]] = int(item[1])
else:
# no config file found
defaults = DEFAULTS
# Build parser for remaining args on command line
parser = argparse.ArgumentParser(
# Don't surpress add_help here so it will handle -h
# Inherit options from config_parser
parents=[preparser],
# print script description with -h/--help
description=__doc__,
# sort options alphabetically
formatter_class=SortingHelpFormatter)
parser.set_defaults(**defaults)
parser.add_argument('infile',
nargs='?',
type=FileType('r', newline=''),
default=sys.stdin,
help=('input filename or stdin in CSV syntax'
' (default: {0})'.format('stdin')))
parser.add_argument('outfile',
nargs='?',
type=FileType('w', encoding='utf-8'),
default=sys.stdout,
help=('output filename or stdout in Ledger syntax'
' (default: {0})'.format('stdout')))
parser.add_argument('--encoding',
metavar='STR',
help=('encoding of csv file'
' (default: {0})'.format(DEFAULTS.encoding)))
parser.add_argument('--ledger-file',
'-l',
metavar='FILE',
help=('ledger file where to read payees/accounts'
' (default search order: {0})'.format(', '.join(
FILE_DEFAULTS.ledger_file))))
parser.add_argument('--quiet',
'-q',
action='store_true',
help=('do not prompt if account can be deduced'
' (default: {0})'.format(DEFAULTS.quiet)))
parser.add_argument('--default-expense',
metavar='STR',
help=('ledger account used as destination'
' (default: {0})'.format(
DEFAULTS.default_expense)))
parser.add_argument('--skip-lines',
metavar='INT',
type=int,
help=('number of lines to skip from CSV file'
' (default: {0})'.format(DEFAULTS.skip_lines)))
parser.add_argument('--reverse',
action='store_true',
help=('reverse the order of entries in the CSV file'
' (default: {0})'.format(DEFAULTS.reverse)))
parser.add_argument('--cleared-character',
choices='*! ',
help=('character to clear a transaction'
' (default: {0})'.format(
DEFAULTS.cleared_character)))
parser.add_argument('--date',
metavar='INT',
type=int,
help=('CSV column number matching date'
' (default: {0})'.format(DEFAULTS.date)))
parser.add_argument('--effective-date',
metavar='INT',
type=int,
help=('CSV column number matching effective date'
' (default: {0})'.format(
DEFAULTS.effective_date)))
parser.add_argument('--desc',
metavar='STR',
help=('CSV column number matching description'
' (default: {0})'.format(DEFAULTS.desc)))
parser.add_argument('--debit',
metavar='INT',
type=int,
help=('CSV column number matching debit amount'
' (default: {0})'.format(DEFAULTS.debit)))
parser.add_argument('--credit',
metavar='INT',
type=int,
help=('CSV column number matching credit amount'
' (default: {0})'.format(DEFAULTS.credit)))
parser.add_argument('--csv-date-format',
metavar='STR',
help=('date format in CSV input file'
' (default: {0})'.format(
DEFAULTS.csv_date_format)))
parser.add_argument('--ledger-date-format',
metavar='STR',
help=('date format for ledger output file'
' (default: {0})'.format(
DEFAULTS.ledger_date_format)))
parser.add_argument('--currency',
metavar='STR',
help=('the currency of amounts'
' (default: {0})'.format(DEFAULTS.currency)))
parser.add_argument('--csv-decimal-comma',
action='store_true',
help=('comma as decimal separator in the CSV'
' (default: {0})'.format(
DEFAULTS.csv_decimal_comma)))
parser.add_argument('--ledger-decimal-comma',
action='store_true',
help=('comma as decimal separator in the ledger'
' (default: {0})'.format(
DEFAULTS.ledger_decimal_comma)))
parser.add_argument('--accounts-file',
metavar='FILE',
help=('file which holds a list of account names'
' (default search order: {0})'.format(', '.join(
FILE_DEFAULTS.accounts_file))))
parser.add_argument('--mapping-file',
metavar='FILE',
help=('file which holds the mappings'
' (default search order: {0})'.format(', '.join(
FILE_DEFAULTS.mapping_file))))
parser.add_argument('--template-file',
metavar='FILE',
help=('file which holds the template'
' (default search order: {0})'.format(', '.join(
FILE_DEFAULTS.template_file))))
parser.add_argument('--tags',
'-t',
action='store_true',
help=('prompt for transaction tags'
' (default: {0})'.format(DEFAULTS.tags)))
parser.add_argument('--clear-screen',
'-C',
action='store_true',
help=('clear screen for every transaction'
' (default: {0})'.format(DEFAULTS.clear_screen)))
parser.add_argument('--delimiter',
metavar='STR',
type=decode_escape_sequences,
help=('delimiter between fields in the csv'
' (default: {0})'.format(DEFAULTS.delimiter)))
parser.add_argument(
'--skip-older-than',
metavar='INT',
type=int,
help=('skip entries more than X days old (-1 indicates keep all)'
' (default: {0})'.format(DEFAULTS.skip_older_than)))
args = parser.parse_args(remaining_argv)
args.ledger_file = find_first_file(args.ledger_file,
FILE_DEFAULTS.ledger_file)
args.mapping_file = find_first_file(args.mapping_file,
FILE_DEFAULTS.mapping_file)
args.accounts_file = find_first_file(args.accounts_file,
FILE_DEFAULTS.accounts_file)
args.template_file = find_first_file(args.template_file,
FILE_DEFAULTS.template_file)
if args.ledger_date_format and not args.csv_date_format:
print(
'csv_date_format must be set'
' if ledger_date_format is defined.',
file=sys.stderr)
sys.exit(1)
if args.encoding != args.infile.encoding:
args.infile = io.TextIOWrapper(args.infile.detach(),
encoding=args.encoding)
return args
def init_conf():
global staging_folder
global destination
global backup_folder
global log_folder
global temp_folder
global running_flag
#print ("do initialization")
config = configparser.RawConfigParser(allow_no_value=True)
if os.path.isfile(".\\static_deploy.conf"):
config.read(".\\static_deploy.conf")
else:
app_1 = wx.App(False)
frame_1 = wx.Frame(None, -1, 'win.py', size=(550, 380))
dlg_11 = wx.MessageDialog(
frame_1, 'no config file found , please add it, exit now ', '',
wx.OK | wx.CANCEL | wx.ICON_ERROR)
val = dlg_11.ShowModal()
dlg_11.Show()
sys.exit(0)
#config.read("D:\\opt\\python_src\\test\\static_deploy.conf")
running_flag = config.get('basic_info', 'running_flag')
logging.basicConfig(level=logging.DEBUG)
if not os.path.isfile(running_flag):
config.read(".\\static_deploy.conf")
#config.read("D:\\opt\\python_src\\test\\static_deploy.conf")
staging_folder = config.get('basic_info', 'staging_folder')
destination = config.get('basic_info', 'destination')
backup_folder = config.get('basic_info', 'backup_folder')
log_folder = config.get('basic_info', 'log_folder')
temp_folder = config.get('basic_info', 'temp_folder')
testing_2 = config.get('basic_info', 'Apps_list')
ff1 = open(running_flag, "w+")
ff1.write("we are running")
ff1.close
for aa in testing_2.split(','):
#print (aa)
path_item1 = staging_folder + "\\" + aa
path_item2 = backup_folder + "\\" + aa
path_item3 = log_folder + "\\" + aa
try:
os.makedirs(path_item1)
except OSError:
pass
try:
#print (path_item2)
os.makedirs(path_item2)
except OSError:
pass
try:
os.makedirs(path_item3)
except OSError:
pass
#try:
# os.stat(path_item2)
#except:
#os.mkdir(path_item2)
else:
#print (" already started , to be safe , we can't re run" )
app1 = wx.App(False)
frame = wx.Frame(None, -1, 'win.py', size=(350, 200))
dlg11 = wx.MessageDialog(frame, 'already started,only once allowed ',
'', wx.OK | wx.CANCEL | wx.ICON_ERROR)
val = dlg11.ShowModal()
dlg11.Show()
sys.exit(0)
import os
import telepot
import configparser
import time
from telepot.loop import MessageLoop
config_cfg = "./config.cfg"
config_raw = configparser.RawConfigParser()
config_raw.read(config_cfg)
debug = config_raw.get("Defaults", "debug_mode")
telepot.api.set_proxy(
"http://190.103.178.44:80") #此行用于解决俄罗斯VPS无法连接上TG服务器的问题，如果你的VPS在其他地区可删除
bot = telepot.Bot('')
print("The bot was loaded compeletly.")
def handle(msg):
debug = config_raw.get("Defaults", "debug_mode")
print('recevied a message!')
content_type, chat_type, chat_id = telepot.glance(msg)
if debug == "True":
print(msg)
return
text = msg['text']
if text == "!debug":
if debug == "False":
if msg['chat']['username'] == "ddhello":
bot.sendMessage(chat_id, "调试模式已开启")
print(debug)
config_raw.set("Defaults", "debug_mode", "True")
else:
if __name__ == "__main__":
parent_parser = ArgumentParser(
description = 'Create pull requests to mumble from transifex translation updates',
epilog = __doc__)
parent_parser.add_argument('-c', '--config', help = 'Configuration file (default: %(default)s)', default = '/etc/prfromtransifex.ini')
parent_parser.add_argument('--setup', help = "If set sets up needed git clone and then exits", action='store_true')
parent_parser.add_argument('-v', '--verbose', help = 'Verbose logging', action='store_true')
args = parent_parser.parse_args()
basicConfig(level = (DEBUG if args.verbose else INFO),
format='%(asctime)s %(levelname)s %(message)s')
debug("Loading configuration from: %s", args.config)
cfg = configparser.RawConfigParser(comment_prefixes=(';'))
cfg.read(args.config)
user = cfg.get('github', 'user')
if cfg.has_option('github', 'password'):
error('Use of passwords is deprecated. Configure a PAT instead.')
sys.exit(1)
pat = cfg.get('github', 'pat')
email = cfg.get('github', 'email')
mode = cfg.get('transifex', 'mode')
minpercent = cfg.get('transifex', 'minpercent')
wr_owner = cfg.get('workingrepo', 'owner')
wr_repo = cfg.get('workingrepo', 'repo')
# -*- coding: utf-8 -*-
import libs.ana_class_atest_log as log
from datetime import datetime as dt
from datetime import timedelta as td
import configparser
config = configparser.RawConfigParser(allow_no_value=True)
config.read('./conf/ana_class_atest.cfg')
def now(string=False):
r = dt.now()
if string:
r = str(r)
return r
def today(string=False):
r = dt.now().date()
if string:
r = str(r)
return r
def day_of_the_week(date=dt.now(), string=False):
r = date.weekday()
if string:
def init(config_filename='config.ini'):
global config
Base.metadata.create_all(engine)
if config is None:
config = configparser.RawConfigParser()
config.read(config_filename)
def create_config_file(config_file_path, run_id):
config = configparser.RawConfigParser()
config.add_section('GENERAL')
config.set('GENERAL', 'workflow', 'AuReMe')
#config.set('GENERAL', 'new_network', '%(network)s')
config.add_section('DATABASE_PATHS')
config.set('DATABASE_PATHS', 'root_db', run_id)
config.set('DATABASE_PATHS', '#data_base', '%(root_db)s/database/XXX')
config.set('DATABASE_PATHS', 'data_base',
'/home/data/database/BIOCYC/METACYC/23.0/metacyc_23.0')
config.set('DATABASE_PATHS', 'mnx_folder', '/home/data/database/MNX/2018')
config.set('DATABASE_PATHS', 'mnx_rxn', '%(mnx_folder)s/reac_xref.tsv')
config.set('DATABASE_PATHS', 'mnx_cpd', '%(mnx_folder)s/chem_xref.tsv')
config.set('DATABASE_PATHS', 'mnx_cpd_prop',
'%(mnx_folder)s/chem_prop.tsv')
config.add_section('PATHS_IN_RUN')
config.set('PATHS_IN_RUN', 'root_path', run_id)
config.set('PATHS_IN_RUN', 'base', os.path.basename(run_id))
config.set('PATHS_IN_RUN', 'networks_folder', '%(root_path)s/networks')
config.set(
'PATHS_IN_RUN', 'annotation_output_folder',
'%(root_path)s/networks/output_annotation_based_reconstruction')
config.set('PATHS_IN_RUN', 'orthology_output_folder',
'%(root_path)s/networks/output_orthology_based_reconstruction')
config.set('PATHS_IN_RUN', 'external_folder',
'%(root_path)s/networks/external_network')
config.set('PATHS_IN_RUN', 'orthology_model_folder',
'%(root_path)s/orthology_based_reconstruction')
config.set('PATHS_IN_RUN', 'orthofinder_workdir',
'%(orthology_model_folder)s/orthofinder_wd')
config.set('PATHS_IN_RUN', 'orthofinder_output',
'%(orthofinder_workdir)s/Orthologues')
config.set('PATHS_IN_RUN', 'dict_gene', '%(model_folder)s/dict_genes.txt')
config.set('PATHS_IN_RUN', 'annotation_folder',
'%(root_path)s/annotation_based_reconstruction')
config.set('PATHS_IN_RUN', 'curation_data_folder',
'%(root_path)s/manual_curation')
config.set('PATHS_IN_RUN', 'genomic_folder', '%(root_path)s/genomic_data')
config.set('PATHS_IN_RUN', 'wiki_pages',
'%(root_path)s/analysis/wiki_pages')
config.set('PATHS_IN_RUN', 'report_dir', '%(root_path)s/analysis/report')
config.set('PATHS_IN_RUN', 'askomics', '%(root_path)s/analysis/askomics')
config.set('PATHS_IN_RUN', 'faa_study', '%(genomic_folder)s/%(base)s.faa')
config.set('PATHS_IN_RUN', 'gbk_study', '%(genomic_folder)s/%(base)s.gbk')
config.set('PATHS_IN_RUN', 'artefacts', 'growth_medium/artefacts')
config.set(
'PATHS_IN_RUN', 'pathwaytools_output',
'%(networks_folder)s/output_annotation_based_reconstruction/pathwaytools/output_pathwaytools'
)
config.set('PATHS_IN_RUN', 'seeds', 'growth_medium/seeds')
config.set('PATHS_IN_RUN', 'seeds_artefacts',
'growth_medium/seeds_artefacts')
config.set('PATHS_IN_RUN', 'targets', 'targets_compounds/targets')
config.set('PATHS_IN_RUN', 'meneco_seeds', '%(seeds)s')
config.set(
'PATHS_IN_RUN', 'meneco_original_output',
'%(base)s/gapfilling/original_output/meneco_output_%(base)s.txt')
config.set(
'PATHS_IN_RUN', 'meneco_solution',
'%(root_path)s/gapfilling/gapfilling_solution_with_meneco_%(base)s.csv'
)
config.set('PATHS_IN_RUN', 'draft', '%(networks_folder)s/draft')
config.add_section('TOOL_PATHS')
config.set('TOOL_PATHS', 'programs', '/programs')
config.set('TOOL_PATHS', 'padmet_u',
'%(programs)s/padmet-utils/padmet_utils')
config.set('TOOL_PATHS', 'reaction_to_add_delete',
'manual_curation/data/reaction_to_add_delete.csv')
config.set('TOOL_PATHS', 'new_reaction_data',
'manual_curation/data/reaction_creator.csv')
config.add_section('TOOL_PARAMETERS')
config.set('TOOL_PARAMETERS', 'orthology_method', 'orthofinder')
config.set('TOOL_PARAMETERS', 'annotation_method', 'pathwaytools')
config.set('TOOL_PARAMETERS', 'gap_filling_method', 'meneco')
config.set('TOOL_PARAMETERS', 'all_rxn', '-f')
config.set('TOOL_PARAMETERS', 'with_artefacts', 'TRUE')
config.set('TOOL_PARAMETERS', 'pwytools_installed', 'FALSE')
config.set('TOOL_PARAMETERS', 'no_orphan', '--no-orphan')
config.set('TOOL_PARAMETERS', 'cutoff', '0.70')
config.set('TOOL_PARAMETERS', 'remove_ortho_workdir', 'TRUE')
config.set('TOOL_PARAMETERS', 'to_map', 'reaction')
config.set('TOOL_PARAMETERS', 'lvl', '3')
# Writing our configuration file to 'example.cfg'
with open(config_file_path, 'w') as configfile:
config.write(configfile)
import smtplib
import configparser
import csv
import copy
import random
from email.mime.text import MIMEText
cronies = []
allMatches = []
delMatches = []
ianMatches = []
# Snag the email props
emailConfig = configparser.RawConfigParser()
emailConfigFilePath = r'email.settings'
emailConfig.read(emailConfigFilePath)
# Fetch the data from the CSV and place it into an array that we'll use to generate matches
with open('secret_santa_responses.csv', newline='') as secretSantaFile:
secretSantaResponses = csv.DictReader(secretSantaFile, delimiter=',')
for crony in secretSantaResponses:
cronies.append({
'Name':
crony['Who are you?'],
'Address':
crony['What address should your present be delivered to?'],
'Naughty or Nice':
crony['Have you been naughty or nice this year?'],
'Jokes':
crony['How do you feel about presents which are inside jokes?'],
'Hobbies':
def run_pants_with_workdir_without_waiting(self, command, workdir, config=None, extra_env=None,
build_root=None, print_exception_stacktrace=True,
**kwargs) -> PantsJoinHandle:
args = [
'--no-pantsrc',
f'--pants-workdir={workdir}',
f'--print-exception-stacktrace={print_exception_stacktrace}',
]
# TODO: If the default value for `--v1` changes to False then this check will
# Have to change to `if '--v1' in command:`.
if '--no-v1' not in command:
args.append('--kill-nailguns')
if self.hermetic():
args.extend(['--pants-config-files=[]',
# Turn off cache globally. A hermetic integration test shouldn't rely on cache,
# or we have no idea if it's actually testing anything.
'--no-cache-read', '--no-cache-write',
# Turn cache on just for tool bootstrapping, for performance.
'--cache-bootstrap-read', '--cache-bootstrap-write'
])
if self.use_pantsd_env_var():
args.append("--enable-pantsd=True")
args.append("--no-shutdown-pantsd-after-run")
if config:
config_data = config.copy()
# TODO(#6071): RawConfigParser is legacy. Investigate updating to modern API.
ini = configparser.RawConfigParser(defaults=config_data.pop('DEFAULT', None))
for section, section_config in config_data.items():
ini.add_section(section)
for key, value in section_config.items():
ini.set(section, key, value)
ini_file_name = os.path.join(workdir, 'pants.ini')
with safe_open(ini_file_name, mode='w') as fp:
ini.write(fp)
args.append('--pants-config-files=' + ini_file_name)
pants_script = [sys.executable, '-m', 'pants']
# Permit usage of shell=True and string-based commands to allow e.g. `./pants | head`.
if kwargs.get('shell') is True:
assert not isinstance(command, list), 'must pass command as a string when using shell=True'
pants_command = ' '.join([*pants_script, ' '.join(args), command])
else:
pants_command = pants_script + args + command
# Only whitelisted entries will be included in the environment if hermetic=True.
if self.hermetic():
env = dict()
# With an empty environment, we would generally get the true underlying system default
# encoding, which is unlikely to be what we want (it's generally ASCII, still). So we
# explicitly set an encoding here.
env['LC_ALL'] = 'en_US.UTF-8'
for h in self.hermetic_env_whitelist():
value = os.getenv(h)
if value is not None:
env[h] = value
hermetic_env = os.getenv('HERMETIC_ENV')
if hermetic_env:
for h in hermetic_env.strip(',').split(','):
value = os.getenv(h)
if value is not None:
env[h] = value
else:
env = os.environ.copy()
if extra_env:
env.update(extra_env)
env.update(PYTHONPATH=os.pathsep.join(sys.path))
# Pants command that was called from the test shouldn't have a parent.
if 'PANTS_PARENT_BUILD_ID' in env:
del env['PANTS_PARENT_BUILD_ID']
# Don't overwrite the profile of this process in the called process.
# Instead, write the profile into a sibling file.
if env.get('PANTS_PROFILE'):
prof = f"{env['PANTS_PROFILE']}.{self._get_profile_disambiguator()}"
env['PANTS_PROFILE'] = prof
# Make a note the subprocess command, so the user can correctly interpret the profile files.
with open(f'{prof}.cmd', 'w') as fp:
fp.write(' '.join(pants_command))
return PantsJoinHandle(
command=pants_command,
process=subprocess.Popen(
pants_command,
env=env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**kwargs
),
workdir=workdir
)
def getAMQPURL():
config = configparser.RawConfigParser()
config.read(CONFIG_FILE_PATH)
return config.get('Messaging', 'CLOUDAMQP_URL')
if new_wifi != '' and new_pw != '' and new_addr != '' and new_port != '':
print("Got config, connecting!")
cparser[cvar.CONFIG_SECTION][cvar.CONFIG_SERVER_STATUS] = 'no_resp'
cvar.write_config(cparser)
return True
time.sleep(2)
if __name__ == '__main__':
print('Controller started.')
print('Switching to AP.')
switch_to_ap()
print('Reading configuration.')
ctr_config = configparser.RawConfigParser()
x = ctr_config.read(cvar.CONFIG_FILENAME)
if len(x) == 0:
ctr_config[cvar.CONFIG_SECTION] = {
cvar.CONFIG_CURR_SSID: '',
cvar.CONFIG_CURR_PW: '',
cvar.CONFIG_SERV_ADDR: '',
cvar.CONFIG_SERV_PORT: '',
cvar.CONFIG_SERVER_STATUS: 'no_conn'
}
cvar.write_config(ctr_config)
if platform.system().lower() != "windows":
os.system('sudo chmod 777 ' + cvar.CONFIG_FILENAME)
print("Configuration no found, writing new.")
# connect to WiFi
def configure_environment(config_filename, environment):
"""
Reads a configuration file from the given path and configures the
given environment accordingly.
"""
factory = environment.factory
if not os.path.exists(config_filename):
raise PysmtIOError("File '%s' does not exists." % config_filename)
# We do not use variable inside the config file
config = cp.RawConfigParser()
config.read(config_filename)
new_solvers_sections = [
s for s in config.sections() if s.lower().startswith("smtlibsolver ")
]
for s in new_solvers_sections:
name = s[len("smtlibsolver "):]
cmd = config.get(s, "command")
assert cmd is not None, ("Missing 'command' value in definition"
"of '%s' solver" % name)
logics_string = config.get(s, "logics")
if logics_string is None:
warn("Missing 'logics' value in definition of '%s' solver" % name,
stacklevel=2)
continue
logics = [get_logic_by_name(l) for l in logics_string.split()]
factory.add_generic_solver(name, cmd.split(), logics)
if "global" in config.sections():
infix = config.get("global", "use_infix_notation")
pref_list = config.get("global", "solver_preference_list")
if infix is not None:
if infix.lower() == "true":
environment.enable_infix_notation = True
elif infix.lower() == "false":
environment.enable_infix_notation = True
else:
warn("Unknown value for 'use_infix_notation': %s" % infix,
stacklevel=2)
if pref_list is not None:
prefs = pref_list.split()
for s in prefs:
if s not in factory.all_solvers():
warn("Unknown solver '%s' in solver_preference_list" % s,
stacklevel=2)
for s in factory.all_solvers():
if s not in prefs:
warn("Solver '%s' is not in the preference list, "\
"and will be disabled." % s,
stacklevel=2)
factory.set_solver_preference_list(prefs)
def setUp(self):
self.config = configparser.RawConfigParser()
self.config.read_dict(fixtures.aws_credentials_conf)
self.account = self.config.sections()[1]
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import configparser
import ibm_whcs_sdk.insights_for_medical_literature as wh
from ibm_cloud_sdk_core.authenticators.no_auth_authenticator import NoAuthAuthenticator
CONFIG = configparser.RawConfigParser()
CONFIG.read('./ibm_whcs_sdk/insights_for_medical_literature/tests/config.ini')
BASE_URL = CONFIG.get('settings', 'base_url')
APIKEY = CONFIG.get('settings', 'key')
IAMURL = CONFIG.get('settings', 'iam_URL')
LEVEL = CONFIG.get('settings', 'logging_level')
VERSION = CONFIG.get('settings', 'version')
DISABLE_SSL = CONFIG.get('settings', 'disable_ssl')
CORPUS = CONFIG.get('custom', 'custom_corpus')
IML_TEST = wh.InsightsForMedicalLiteratureServiceV1(
authenticator=NoAuthAuthenticator(), version=VERSION)
IML_TEST.set_service_url(BASE_URL)
def parse_config_file(options: Options, filename: Optional[str]) -> None:
"""Parse a config file into an Options object.
Errors are written to stderr but are not fatal.
If filename is None, fall back to default config file and then
to setup.cfg.
"""
if filename is not None:
config_files = (filename, ) # type: Tuple[str, ...]
else:
config_files = (defaults.CONFIG_FILE, ) + SHARED_CONFIG_FILES
parser = configparser.RawConfigParser()
for config_file in config_files:
if not os.path.exists(config_file):
continue
try:
parser.read(config_file)
except configparser.Error as err:
print("%s: %s" % (config_file, err), file=sys.stderr)
else:
file_read = config_file
options.config_file = file_read
break
else:
return
if 'mypy' not in parser:
if filename or file_read not in SHARED_CONFIG_FILES:
print("%s: No [mypy] section in config file" % file_read,
file=sys.stderr)
else:
section = parser['mypy']
prefix = '%s: [%s]' % (file_read, 'mypy')
updates, report_dirs = parse_section(prefix, options, section)
for k, v in updates.items():
setattr(options, k, v)
options.report_dirs.update(report_dirs)
for name, section in parser.items():
if name.startswith('mypy-'):
prefix = '%s: [%s]' % (file_read, name)
updates, report_dirs = parse_section(prefix, options, section)
if report_dirs:
print(
"%s: Per-module sections should not specify reports (%s)" %
(prefix, ', '.join(s + '_report'
for s in sorted(report_dirs))),
file=sys.stderr)
if set(updates) - Options.PER_MODULE_OPTIONS:
print(
"%s: Per-module sections should only specify per-module flags (%s)"
% (prefix, ', '.join(
sorted(set(updates) - Options.PER_MODULE_OPTIONS))),
file=sys.stderr)
updates = {
k: v
for k, v in updates.items()
if k in Options.PER_MODULE_OPTIONS
}
globs = name[5:]
for glob in globs.split(','):
# For backwards compatibility, replace (back)slashes with dots.
glob = glob.replace(os.sep, '.')
if os.altsep:
glob = glob.replace(os.altsep, '.')
pattern = re.compile(fnmatch.translate(glob))
options.per_module_options[pattern] = updates
options.unused_configs[pattern] = glob
def _build_config_wrapper(self, data, filename):
fileconfig = configparser.RawConfigParser(strict=False)
self._parse_config(data, filename, fileconfig, set())
return ConfigWrapper(self.printer, fileconfig, {}, 'printer')
OBJECTS_LIST = []
SECTIONS_LIST = []
FILES_LIST = []
ALL_CHECKS = []
for i in range(len(CHECKFILES)):
E = collections.OrderedDict()
E = {"{#CHECKS_FILE}": i}
FILES_LIST.append(E)
FILES_JSON = '{\"data\":' + json.dumps(FILES_LIST) + '}'
to_outfile(config, ME + ".files.lld", FILES_JSON)
for i in range(3, len(CHECKFILES)):
# #0 is executable that is also checked for updates
# #1 dbc module
# #2 dbe module
CHECKS = configparser.RawConfigParser()
try:
CHECKSF = open(CHECKFILES[i]['name'], 'r')
to_outfile(config, ME + "[checks," + str(i) + \
",name]", CHECKFILES[i]['name'])
to_outfile(config, ME + "[checks," + str(i) + \
",lmod]",
str(int(os.stat(CHECKFILES[i]['name']).st_mtime)))
try:
CHECKS.read_file(CHECKSF)
CHECKSF.close()
to_outfile(config, ME + "[checks," + str(i) + \
",status]", 0)
except configparser.Error:
to_outfile(config, ME + "[checks," + str(i) + \
",status]", 13)
def parse_config_file_set_env():
config = configparser.RawConfigParser()
files_read = config.read('../test_suite_config.ini')
if len(files_read) != 1:
raise Exception("Failed to find/open test_suite_config.ini file")
# get the platform type since config file has property for platform respectively.
os_type = platform.system()
os_type = os_type.upper()
# check if the config are defined for current os type.
platform_list = config.sections()
try:
platform_list.index(os_type)
except:
raise Exception("not able to find the config defined for ostype " +
os_type)
# set all the environment variables
# TEST_DIRECTORY_PATH is the location where test_data folder will be created and test files will be created further.
# set the environment variable TEST_DIRECTORY_PATH
os.environ['TEST_DIRECTORY_PATH'] = config[os_type]['TEST_DIRECTORY_PATH']
# AZCOPY_EXECUTABLE_PATH is the location of the azcopy executable
# azcopy executable will be copied to test data folder.
# set the environment variables
os.environ['AZCOPY_EXECUTABLE_PATH'] = config[os_type][
'AZCOPY_EXECUTABLE_PATH']
# TEST_SUITE_EXECUTABLE_LOCATION is the location of the test suite executable
# test suite executable will be copied to test data folder.
# set the environment variable TEST_SUITE_EXECUTABLE_LOCATION
os.environ['TEST_SUITE_EXECUTABLE_LOCATION'] = config[os_type][
'TEST_SUITE_EXECUTABLE_LOCATION']
# CONTAINER_SAS_URL is the shared access signature of the container
# where test data will be uploaded to and downloaded from.
os.environ['CONTAINER_SAS_URL'] = config['CREDENTIALS'][
'CONTAINER_SAS_URL']
# container whose storage account has been configured properly for the interactive testing user.
os.environ['CONTAINER_OAUTH_URL'] = config['CREDENTIALS'][
'CONTAINER_OAUTH_URL']
# container which should be same to CONTAINER_OAUTH_URL, while with SAS for validation purpose.
os.environ['CONTAINER_OAUTH_VALIDATE_SAS_URL'] = config['CREDENTIALS'][
'CONTAINER_OAUTH_VALIDATE_SAS_URL']
# share_sas_url is the URL with SAS of the share where test data will be uploaded to and downloaded from.
os.environ['SHARE_SAS_URL'] = config['CREDENTIALS']['SHARE_SAS_URL']
# container sas of the premium storage account.
os.environ['PREMIUM_CONTAINER_SAS_URL'] = config['CREDENTIALS'][
'PREMIUM_CONTAINER_SAS_URL']
# set the account name for blob fs service operation
os.environ['ACCOUNT_NAME'] = config['CREDENTIALS']['ACCOUNT_NAME']
# set the account key for blob fs service operation
os.environ['ACCOUNT_KEY'] = config['CREDENTIALS']['ACCOUNT_KEY']
# set the filesystem url in the environment
os.environ['FILESYSTEM_URL'] = config['CREDENTIALS']['FILESYSTEM_URL']
os.environ['FILESYSTEM_SAS_URL'] = config['CREDENTIALS'][
'FILESYSTEM_SAS_URL']
# set env var for service-2-service copy source blob account
os.environ['S2S_SRC_BLOB_ACCOUNT_SAS_URL'] = config['CREDENTIALS'][
'S2S_SRC_BLOB_ACCOUNT_SAS_URL']
# set env var for service-2-service copy destination blob account
os.environ['S2S_DST_BLOB_ACCOUNT_SAS_URL'] = config['CREDENTIALS'][
'S2S_DST_BLOB_ACCOUNT_SAS_URL']
# set env var for service-2-service copy source file account
os.environ['S2S_SRC_FILE_ACCOUNT_SAS_URL'] = config['CREDENTIALS'][
'S2S_SRC_FILE_ACCOUNT_SAS_URL']
# set env var for service-2-service copy source s3 and gcp
os.environ['S2S_SRC_S3_SERVICE_URL'] = config['CREDENTIALS'][
'S2S_SRC_S3_SERVICE_URL']
os.environ['S2S_SRC_GCP_SERVICE_URL'] = config['CREDENTIALS'][
'S2S_SRC_GCP_SERVICE_URL']
os.environ['AWS_ACCESS_KEY_ID'] = config['CREDENTIALS'][
'AWS_ACCESS_KEY_ID']
os.environ['AWS_SECRET_ACCESS_KEY'] = config['CREDENTIALS'][
'AWS_SECRET_ACCESS_KEY']
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = config['CREDENTIALS'][
'GOOGLE_APPLICATION_CREDENTIALS']
os.environ['GOOGLE_CLOUD_PROJECT'] = config['CREDENTIALS'][
'GOOGLE_CLOUD_PROJECT']
os.environ['OAUTH_AAD_ENDPOINT'] = config['CREDENTIALS'][
'OAUTH_AAD_ENDPOINT']
os.environ['S3_TESTS_OFF'] = config['CREDENTIALS']['S3_TESTS_OFF']
os.environ['GCP_TESTS_OFF'] = config['CREDENTIALS']['GCP_TESTS_OFF']
# encoding: utf-8 ''' @author: ZYZ @contact: [email protected] @file: Main.py @time: 5/9/2020 19:10 @desc:半自动淘宝商品评论抓取工具 ''' import taobao import filePreRegular import dataAnalysis as DA import configparser import time import random cp = configparser.RawConfigParser() cp.read('Infor.conf') #1 print('数据爬取开始') print('-'*20) for i in range(0,int(cp.get('taobao','pageNumber'))): filename = cp.get('taobao','name')+str(i) filePreRegular.fileInput(taobao.crawlerTaobao(i+1),filename) list = filePreRegular.fileProcess(filename) filePreRegular.fileOutput(list,filename) print("已爬取第{}页评论".format(i)) time.sleep(random.randint(20,30)) # #2 # print('数据获取成功，开始生成报表')
def create_profile(self):
profile = Munch(name='BBTZ5TEST')
profile.path = os.path.expanduser(f'~/.{profile.name}')
profile.profiles = {
# 'Linux': os.path.expanduser(f'~/.{self.client}/{self.client}'),
'Linux':
os.path.expanduser(f'~/.{self.client}/zotero'),
'Darwin':
os.path.expanduser('~/Library/Application Support/' + {
'zotero': 'Zotero',
'jurism': 'Juris-M'
}[self.client]),
}[platform.system()]
os.makedirs(profile.profiles, exist_ok=True)
beta = ''
if self.beta: beta = '-beta'
profile.binary = {
'Linux':
f'/usr/lib/{self.client}{beta}/{self.client}',
'Darwin':
f'/Applications/{self.client.title()}{beta}.app/Contents/MacOS/{self.client}',
}[platform.system()]
# create profile
profile.ini = os.path.join(profile.profiles, 'profiles.ini')
ini = configparser.RawConfigParser()
ini.optionxform = str
if os.path.exists(profile.ini): ini.read(profile.ini)
if not ini.has_section('General'): ini.add_section('General')
profile.id = None
for p in ini.sections():
for k, v in ini.items(p):
if k == 'Name' and v == profile.name: profile.id = p
if not profile.id:
free = 0
while True:
profile.id = f'Profile{free}'
if not ini.has_section(profile.id): break
free += 1
ini.add_section(profile.id)
ini.set(profile.id, 'Name', profile.name)
ini.set(profile.id, 'IsRelative', 0)
ini.set(profile.id, 'Path', profile.path)
ini.set(profile.id, 'Default', None)
with open(profile.ini, 'w') as f:
ini.write(f, space_around_delimiters=False)
# layout profile
if self.config.profile:
profile.firefox = webdriver.FirefoxProfile(
os.path.join(ROOT, 'test/db', self.config.profile))
profile.firefox.set_preference(
'extensions.zotero.dataDir',
os.path.join(profile.path, self.client))
profile.firefox.set_preference('extensions.zotero.useDataDir',
True)
profile.firefox.set_preference(
'extensions.zotero.translators.better-bibtex.removeStock',
False)
else:
profile.firefox = webdriver.FirefoxProfile(
os.path.join(ROOT, 'test/fixtures/profile', self.client))
self.install_xpis(os.path.join(ROOT, 'xpi'), profile.firefox)
self.install_xpis(os.path.join(ROOT, 'other-xpis'), profile.firefox)
if self.config.db:
self.install_xpis(
os.path.join(ROOT, 'test/db', self.config.db, 'xpis'),
profile.firefox)
if self.config.profile:
self.install_xpis(
os.path.join(ROOT, 'test/db', self.config.profile, 'xpis'),
profile.firefox)
profile.firefox.set_preference(
'extensions.zotero.translators.better-bibtex.testing',
self.testing)
profile.firefox.set_preference(
'extensions.zotero.translators.better-bibtex.workers',
self.workers)
profile.firefox.set_preference(
'extensions.zotero.debug-bridge.password', self.password)
profile.firefox.set_preference('dom.max_chrome_script_run_time',
self.config.timeout)
utils.print(f'dom.max_chrome_script_run_time={self.config.timeout}')
with open(os.path.join(os.path.dirname(__file__),
'preferences.toml')) as f:
preferences = toml.load(f)
for p, v in nested_dict_iter(preferences['general']):
profile.firefox.set_preference(p, v)
if self.config.locale == 'fr':
for p, v in nested_dict_iter(preferences['fr']):
profile.firefox.firefox.set_preference(p, v)
if not self.config.first_run:
profile.firefox.set_preference(
'extensions.zotero.translators.better-bibtex.citekeyFormat',
'[auth][shorttitle][year]')
if self.client == 'jurism':
utils.print(
'\n\n** WORKAROUNDS FOR JURIS-M IN PLACE -- SEE https://github.com/Juris-M/zotero/issues/34 **\n\n'
)
profile.firefox.set_preference(
'extensions.zotero.dataDir',
os.path.join(profile.path, 'jurism'))
profile.firefox.set_preference('extensions.zotero.useDataDir',
True)
profile.firefox.set_preference(
'extensions.zotero.translators.better-bibtex.removeStock',
False)
profile.firefox.update_preferences()
shutil.rmtree(profile.path, ignore_errors=True)
shutil.move(profile.firefox.path, profile.path)
profile.firefox = None
if self.config.db:
self.needs_restart = True
utils.print(f'restarting using {self.config.db}')
dbs = os.path.join(ROOT, 'test', 'db', self.config.db)
if not os.path.exists(dbs): os.makedirs(dbs)
db_zotero = os.path.join(dbs, f'{self.client}.sqlite')
db_zotero_alt = os.path.join(dbs, self.client,
f'{self.client}.sqlite')
if not os.path.exists(db_zotero) and not os.path.exists(
db_zotero_alt):
urllib.request.urlretrieve(
f'https://github.com/retorquere/zotero-better-bibtex/releases/download/test-database/{self.config.db}.zotero.sqlite',
db_zotero)
if not os.path.exists(db_zotero): db_zotero = db_zotero_alt
shutil.copy(
db_zotero,
os.path.join(profile.path, self.client,
os.path.basename(db_zotero)))
db_bbt = os.path.join(dbs, 'better-bibtex.sqlite')
db_bbt_alt = os.path.join(dbs, self.client, 'better-bibtex.sqlite')
if not os.path.exists(db_bbt) and not os.path.exists(db_bbt_alt):
urllib.request.urlretrieve(
f'https://github.com/retorquere/zotero-better-bibtex/releases/download/test-database/{self.config.db}.better-bibtex.sqlite',
db_bbt)
if not os.path.exists(db_bbt): db_bbt = db_bbt_alt
shutil.copy(
db_bbt,
os.path.join(profile.path, self.client,
os.path.basename(db_bbt)))
# remove any auto-exports that may exist
db = sqlite3.connect(
os.path.join(profile.path, self.client,
os.path.basename(db_bbt)))
ae = None
for (ae, ) in db.execute(
'SELECT data FROM "better-bibtex" WHERE name = ?',
['better-bibtex.autoexport']):
ae = json.loads(ae)
ae['data'] = []
if ae:
db.execute(
'UPDATE "better-bibtex" SET data = ? WHERE name = ?',
[json.dumps(ae), 'better-bibtex.autoexport'])
db.commit()
db.close()
return profile
