def main():
arguments = ArgumentParser().arguments
Utils.make_dir(Config.log_dirpath)
logger = Logger(filename=Config.log_filename, is_debug=arguments.debug)
logger.info("Start crawling... (migrate={}, debug={})".format(
arguments.migrate, arguments.debug))
news_list, df = Crawler(arguments.migrate, logger).execute()
if len(news_list) > 0 and not arguments.migrate:
slack_bot = SlackBot(Config.channel, Config.debug_channel,
Config.slack_api_token)
for news in news_list:
message = Message.make_message(news)
is_success, status = slack_bot.post_message(
message, is_debug=arguments.debug)
if is_success:
logger.info(status)
df.to_csv(Config.database_filename, index=False)
else:
logger.error(status)
logger.info("Stop crawling... (migrate={}, debug={})".format(
arguments.migrate, arguments.debug))
def __init__(self):
args = ArgumentParser()
self.count = args.text_start
self.urls = url_manager.UrlManager()
self.downloader = html_downloader.HtmlDownloader()
self.parser = html_parser.HtmlParser()
self.outputer = outputer.HtmlOutputer()
def init_parser(self):
parser = ArgumentParser()
admin_branch, help_branch, course_branch = \
parser.add_argument('branch', branching=[('admin', lambda x: x == 'admin'), ('help', lambda x: x == 'help'),
('course', lambda x: True)],
help='select course, admin, or help')
course_branch.help = 'Enter your course to submit files to it.'
course_branch.add_argument(
'assignment', help='specify which assignment you are submitting')
course_branch.add_argument(
'files',
multiple=0,
help='specify the files which you are submitting')
admin_branch.add_argument(
'admin-select',
help=
"choose one of: add-admin, remove-admin, create-course, delete-course, display-admins, display-courses"
)
admin_branch.add_argument('names', optional=True, multiple=0)
'''
branching=[('add-admin', lambda x: x == 'add-admin'),
('remove-admin', lambda x: x == 'remove-admin'),
('create-course', lambda x: x == 'create-course'),
('delete-course', lambda x: x == 'delete-course'),
('display-admins', lambda x: x == 'display-admins'),
('display-courses', lambda x: x == 'display-courses')],
'''
return parser
def __init__(self):
self.args = ArgumentParser()
self.file_count = 0
self.log_file = self.args.update_html_log_path + datetime.today().strftime("%Y-%m-%d") + ".txt"
if not os.path.exists(self.log_file):
os.mknod(self.log_file)
self.log_file_handler = open(self.log_file, "a+", encoding='utf-8')
def __init__(self, argv):
arg_parser = ArgumentParser()
arg = arg_parser.parse(sys.argv[1:])
self.nb_person = arg[0]
self.simulation_type = arg[1]
self.metrics_enable = arg[2]
self.scenario = None
def main(args):
host = typ.Host()
runner = typ.Runner(host)
parser = ArgumentParser(host)
parser.prog = os.path.basename(sys.argv[0])
parser.description = __doc__
parser.formatter_class = argparse.RawDescriptionHelpFormatter
runner.parse_args(
parser=parser,
argv=args,
isolate=['installer_test.*'], # InstallerTest must be serialized.
top_level_dir=CUR_DIR,
retry_limit=3, # Retry failures by default since the tests are flaky.
)
if parser.exit_status is not None:
return parser.exit_status
# Stuff args into environment vars for use by child procs.
_prepare_env_for_subprocesses(parser, runner.args)
try:
return runner.run()[0]
except KeyboardInterrupt:
self.print_("interrupted, exiting", stream=self.host.stderr)
return 130
def main():
"""Main function for calling others"""
parsing = ArgumentParser()
rooms_file, students_file = parsing.args_info()
example = FileReader()
rooms = example.file_reader(rooms_file)
students = example.file_reader(students_file)
new_info = Distribution(rooms, students).student_distribution()
result = JsonExporter(new_info).unloading()
print(result)
def __init__(self):
args = ArgumentParser()
keyword_start = args.keyword_start
keyword_end = args.keyword_end
self.new_urls = set()
for index in range(keyword_start, keyword_end):
url = "https://baike.baidu.com/view/" + str(index)
self.new_urls.add(url)
self.old_urls = set()
self.fail_urls = set()
self.fail_url_mark = True
def main():
level = ArgumentParser().start()
if level is not None:
LogHandler(log_level=level)
else:
LogHandler()
process_data = ProcessData()
#process_data.test_jitbit()
process_data.start()
def main():
argument_parser = ArgumentParser()
# どちらも指定されてない or どちらも指定されている場合にどちらも行う
is_both = not argument_parser.arguments.crawl ^ argument_parser.arguments.scrape
if is_both:
Crawler.run()
Scraper.run()
else:
if argument_parser.arguments.crawl:
Crawler.run()
elif argument_parser.arguments.scrape:
Scraper.run()
def __get_argparse_configuration__():
parser = ArgumentParser(description="Log argument parser",
usage=argparse.SUPPRESS)
parser.add_argument("--level",
"-l",
dest="level",
default="Debug",
required=False,
choices=['Warn', 'Info', 'Debug', 'Error'])
parser.add_argument("--message",
"-m",
nargs='+',
dest="message",
required=True)
return parser
def __init__(self):
args = ArgumentParser()
text_start = args.text_start
text_end = args.text_end
self.new_urls = set()
key_words = [
line.strip()
for line in open(args.key_word_path, 'r', encoding='UTF-8')
]
for index in range(text_start, text_end):
url = "https://baike.baidu.com/item/" + quote(
key_words[index].strip())
self.new_urls.add(url)
key_words.clear()
self.old_urls = set()
self.fail_urls = set()
self.fail_url_mark = True
def __init__(self, interrupt=False):
self.args = ArgumentParser()
self.count = self.args.keyword_start
self.urls = url_manager.UrlManager()
self.downloader = html_downloader.HtmlDownloader()
self.parser = html_parser.HtmlParser()
self.outputer = outputer.HtmlOutputer()
self.title_to_id = dict()
self.interrupt = interrupt
# 初始化ES客户端
# self.es = Elasticsearch(hosts=self.args.es_url, timeout=120)
# 爬虫更新中断后保存之前爬取的词条
if self.interrupt:
self.interrupt_previous_titles = set()
self.load_interrupt_titles()
print("Load %d titles." % len(self.interrupt_previous_titles))
def _initialize():
"""Initializes the InstallerTest class.
This entails setting the class attributes and adding the configured test
methods to the class.
"""
args = ArgumentParser().parse_args()
log_level = (logging.ERROR if args.quiet else
logging.DEBUG if args.verbose else logging.INFO)
LOGGER.setLevel(log_level)
handler = logging.StreamHandler()
handler.setFormatter(
logging.Formatter(
fmt='[%(asctime)s:%(filename)s(%(lineno)d)] %(message)s',
datefmt='%m%d/%H%M%S'))
LOGGER.addHandler(handler)
# Pull args from the parent proc out of the environment block.
if os.environ.get('CMI_FORCE_CLEAN', False):
global _force_clean
_force_clean = True
InstallerTest._output_dir = os.environ.get('CMI_OUTPUT_DIR')
installer_path = GetAbsoluteExecutablePath(
os.environ.get('CMI_INSTALLER_PATH', 'mini_installer.exe'))
previous_version_installer_path = GetAbsoluteExecutablePath(
os.environ.get('CMI_PREVIOUS_VERSION_INSTALLER_PATH',
'previous_version_mini_installer.exe'))
chromedriver_path = GetAbsoluteExecutablePath(
os.environ.get('CMI_CHROMEDRIVER_PATH', 'chromedriver.exe'))
config_path = GetAbsoluteConfigPath(
os.environ.get('CMI_CONFIG', 'config.config'))
InstallerTest._variable_expander = VariableExpander(
installer_path, previous_version_installer_path, chromedriver_path,
args.quiet, InstallerTest._output_dir)
InstallerTest._config = ParseConfigFile(config_path,
InstallerTest._variable_expander)
# Add a test_Foo function to the InstallerTest class for each test in
# the config file.
run_test_fn = getattr(InstallerTest, 'run_test')
for test in InstallerTest._config.tests:
setattr(InstallerTest, 'test_' + test['name'], run_test_fn)
def __init__(self):
args = ArgumentParser()
index_start = 1
try:
with open(args.index_end_path, 'r', encoding='utf-8') as f:
index_end = int(f.readline().strip('\n'))
except Exception as e:
print(e)
sys.exit(-1)
self.new_urls = set()
print("Adding all urls ...")
for index in range(index_start, index_end):
url = "https://baike.baidu.com/view/" + str(index)
self.new_urls.add(url)
print("Done.")
self.old_urls = set()
self.fail_urls = set()
self.fail_url_mark = True
self.downloader = HtmlDownloader()
def main():
args = ArgumentParser().args
image_path = args.image_path
address = host + ":" + str(port)
# task_resize_addr = address + '/api/new_task'
one_resize_addr = address + api_image_resize_endpoint
if not is_valid_image(image_path):
print("Error: Invalid image path {}".format(image_path), file=sys.stderr)
exit(1)
new_image_name, image_extension = get_new_image_name(image_path, args.new_name)
files = prepare_image_package_to_send(image_path, args.new_width, args.new_height)
response = requests.post(one_resize_addr, files=files)
nparr = np.frombuffer(response.content, np.uint8)
new_sized_image = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
file_drop_loc = args.drop_loc + new_image_name + "." + image_extension
save_new_resized_image(new_sized_image, file_drop_loc)
def main(args):
a = ArgumentParser()
args = a.parse_args(args)
s = Searcher()
b = Booker()
try:
# 'from' and 'return' are keywords, need to use vars() :-(
token = s.search(
args.date,
vars(args)['from'],
args.to,
cheapest=False if args.fastest else True,
length_of_stay=(0 if args.one_way else vars(args)['return'])
)
pnr = b.book(token, args.bags)
print(pnr)
except BookFlightError as e:
print(e, file=sys.stderr)
def __get_argparse_configuration__():
parser = ArgumentParser(description="Graphics argument parser",
usage=argparse.SUPPRESS)
parser.add_argument("--shape",
"-s",
dest="shape",
choices=["Circle", "Rectangle"],
required=True)
parser.add_argument("--color",
"-c",
type=check_color_variable,
nargs=3,
dest="color")
parser.add_argument("--area",
"-a",
type=int,
nargs=4,
dest="area",
required=True)
return parser
import sys
import os
import time
from download import FileDownloader
from processing import TesseractProcessor, OctaveProcessor
from argument_parser import ArgumentParser, ArgumentGroup, MutexArgumentParser, StaticArgument
args = ArgumentParser.parse_argv([
ArgumentParser("cache_dir", "-cd", 1, "./verser",
lambda params: params[0]),
ArgumentParser("result_dir", "-rd", 1, "./verser",
lambda params: params[0]),
ArgumentParser("processor_id", "-p", 1, "ocr", lambda params: params[0]),
MutexArgumentParser("files", [
ArgumentParser(
"files", "-fseq", 3, None, lambda params: {
"type": "sequence",
"format": params[0],
"offset": int(params[1]),
"count": int(params[2])
}),
ArgumentGroup("files", [
StaticArgument("type", "sequence"),
ArgumentParser("offset", "-o", 1, 1,
lambda params: int(params[0])),
ArgumentParser(
"format", "-furl", 1,
"https://hosting.softagent.se/upps-personverser/PictureLoader?Antialias=ON&ImageId=%s&Scale=1",
lambda params: params[0]),
ArgumentParser("count", "-c", 1, 108953,
def setUp(self):
self.parser = ArgumentParser()
import matplotlib.pyplot as plt
from numpy import flip
from timeit import default_timer as timer
from argument_parser import ArgumentParser
from classifiers.bayes import Bayes
from classifiers.decision_tree import DecisionTree
from classifiers.k_neighbors import KNeighbors
from classifiers.neural_network import NeuralNetwork
from classifiers.svm import SVM
from utils import factorize, load_dataset, learning_curve_plot, print_basic_stats
args = ArgumentParser().get_arguments()
data = load_dataset(args.dataset)
labels = data[data.columns[-1]].unique()
data = data.apply(factorize)
fraction = args.training_fraction
class_args = args.class_args
if args.classifier is 0:
classifiers = [DecisionTree(data, labels, fraction, class_args),
Bayes(data, labels, fraction, class_args),
SVM(data, labels, fraction, class_args),
KNeighbors(data, labels, fraction, class_args),
NeuralNetwork(data, labels, fraction, class_args)]
elif args.classifier is 1:
classifiers = [DecisionTree(data, labels, fraction, class_args)]
elif args.classifier is 2:
classifiers = [Bayes(data, labels, fraction, class_args)]
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ==============================================================================
import gc
import os
from tqdm import tqdm
import torch
from chembl import Chembl
from graphgen import GraphGen
from saver import Saver
from visdom_helper import Plot2D, Image
from argument_parser import ArgumentParser
parser = ArgumentParser(description='Process some integers.')
parser.add_argument("-lr", type=float, default=3e-4)
parser.add_argument("-wd", type=float, default=0)
parser.add_argument("-optimizer", default="adam")
parser.add_argument("-batch_size", type=int, default=128)
parser.add_argument("-save_dir", type=str)
parser.add_argument("-force", type=bool, default=0, save=False)
parser.add_argument("-gpu", type=str, default="")
parser.add_argument("-lr_milestones",
default="none",
parser=ArgumentParser.int_list_parser)
parser.add_argument("-lr_gamma", default=0.3)
parser.add_argument("-dropout", default=0.0)
parser.add_argument("-state_size", default=128)
parser.add_argument("-early_stop", type=bool, default=1)
parser.add_argument("-kekulize", type=bool, default=0)
import sys
from argument_parser import ArgumentParser
from drawer import Drawer
from plotter import Plotter
from sequence import Sequence
class Dotplot(object):
def __init__(self, sequences, plotter_args=None, drawer_args=None):
self.sequences = [
Sequence.from_fasta_file(sequences.file1),
Sequence.from_fasta_file(sequences.file2)
]
self.plotter = Plotter(plotter_args)
self.drawer = Drawer(drawer_args)
def make_plot(self):
self.plot = self.plotter.plot(self.sequences)
def draw(self, plot=None):
if not plot:
plot = self.plot
self.drawer.draw(plot)
if __name__ == '__main__':
args = ArgumentParser().parse(sys.argv)
dotplot = Dotplot(args.sequences, args.plotter, args.drawer)
dotplot.make_plot()
dotplot.draw()
# else:
# self.outputer.collect_data(html_cont, item_url, title)
if self.interrupt and title in self.interrupt_previous_titles:
self.urls.add_old_url(page_url)
self.count += 1
print("Keyword already exist %d Url: %s" % (self.count, unquote(page_url)))
continue
if update_time >= '2018-01-01 00:00:00':
self.outputer.collect_data(html_cont, item_url, title)
self.urls.add_old_url(page_url)
self.count += 1
# if self.count % 1000000 == 0:
# logging.debug('Visited %d items.' % self.count)
print("Keyword Success %d Url:%s" % (self.count, unquote(page_url)))
except:
self.urls.add_fail_url(page_url)
print("Failed")
if __name__ == '__main__':
args = ArgumentParser()
logging.basicConfig(
level=logging.INFO,
filename=args.update_output_log_path,
filemode='a',
format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s'
)
spider = SpiderMain(interrupt=True)
# logging.info("The number of entities with baike urls is {}.".format(len(spider.url_to_update_time)))
spider.craw()
spider.outputer.log_file_handler.close()
def __get_argparse_configuration__():
parser = ArgumentParser(description="Save argument parser", usage=argparse.SUPPRESS)
parser.add_argument("--file","-f", type=str, dest="file", required=False)
return parser
"INSERT INTO ReferenceObjects(uuid, eid, name, ref_type, x, y, z, pitch, yaw) VALUES (?,?,?,?,?,?,?,?,?)",
self.memory.self_memid,
p.entityId,
p.name,
"player",
p.pos.x,
p.pos.y,
p.pos.z,
p.look.pitch,
p.look.yaw,
)
if __name__ == "__main__":
base_path = os.path.dirname(__file__)
parser = ArgumentParser("Minecraft", base_path)
opts = parser.parse()
# set up stdout logging
sh = logging.StreamHandler()
sh.setLevel(logging.DEBUG if opts.verbose else logging.INFO)
sh.setFormatter(log_formatter)
logging.getLogger().addHandler(sh)
logging.info("Info logging")
logging.debug("Debug logging")
# Check that models and datasets are up to date
rc = subprocess.call([opts.verify_hash_script_path, "craftassist"])
set_start_method("spawn", force=True)
from argument_parser import ArgumentParser
from data_loader import load_data
from drift_detectors.drift_adwin import DriftADWIN
from drift_detectors.drift_ddm import DriftDDM
from drift_detectors.drift_eddm import DriftEDDM
from drift_detectors.drift_page_hinkley import DriftPageHinkley
from classifiers.bayes import Bayes
from classifiers.neural_network import NeuralNetwork
from classifiers.knn import KNN
from classifiers.svm import SVM
from util import *
argument_parser = ArgumentParser()
filename = argument_parser.get_filename()
adwin_delta = argument_parser.get_delta()
training_set_ratio = argument_parser.get_training_set_ratio()
neighbors_number = argument_parser.get_neighbors_number()
kernel = argument_parser.get_kernel()
regulation = argument_parser.get_regulation()
max_iters = argument_parser.get_iterations()
n_of_hidden = argument_parser.get_n_of_hidden_layers()
algorithm = argument_parser.get_algorithm()
printing = argument_parser.is_printing()
data, labels = load_data(filename)
classifiers = {
'bayes': Bayes(data, labels, training_set_ratio),
'knn': KNN(data, labels, training_set_ratio, neighbors_number),
'nn': NeuralNetwork(data, labels, training_set_ratio, n_of_hidden,
max_iters),
from log_utils import LogUtils
from merge_utils import MergeUtils
if __name__ == "__main__":
#Init argument parser
a_parser = ArgumentParser([{
"short_name": '-f1',
"long_name": "--folder1",
"required": True,
"help": "Path to first folder"
}, {
"short_name": '-f2',
"long_name": "--folder2",
"required": True,
"help": "Path to second folder"
}, {
"short_name": '-l',
"long_name": "--log",
"required": False,
"help": "Export log as CSV"
}, {
"short_name": '-m',
"long_name": '--merge',
"required": False,
"help": "Merge both folders in one"
}])
args = a_parser.get_args()
image_paths = []
image_paths.append(get_image_paths(args['folder1']))
print("[INFO] Found " + str(len(image_paths[-1])) +
reads_processed, short_reads = self.mh_dict[mh_name].get_alleles(
mh_number_of_bases_targeted)
self.total_reads_count += reads_processed
self.short_reads_count += short_reads
self.mh_dict[mh_name].calculate_avg_mapping_quality()
self.mh_dict[mh_name].calculate_total_coverage()
self.mh_dict[mh_name].filter_alleles_and_set_analytical_threshold(
self.filters_obj)
self.mh_dict[mh_name].calculate_number_of_alleles()
self.mh_dict[mh_name].calculate_number_of_contributors()
if self.info:
self.info_obj.add_info(self.mh_dict[mh_name])
def _get_number_of_contributors_overall(self):
self.number_of_contributors_overall = max([
self.mh_dict[mh_name].number_of_contributors
for mh_name in self.mh_dict.keys()
])
if __name__ == '__main__':
args_obj = ArgumentParser()
mh_obj = Microhaplotyper(bam=args_obj.args.bam_file_path,
bed=args_obj.args.bed_file_path,
info=args_obj.args.info_file_path,
out=args_obj.args.out_file_path,
analytical_threshold=args_obj.args.min_coverage)
mh_obj.get_microhaplotypes()
output_obj = OutputMicrohaplotypes(mh_obj)
output_obj.output_microhaplotypes()
def __init__(self):
self.args = ArgumentParser()
text_start = self.args.text_start
self.file_count = int(text_start / 300000)
self.datas = []