def execute():
parser = OptionParser(usage="%prog config [options]")
parser.add_option("-f",
"--folder",
dest="app_dir",
help="folder where the certs and keys to stay")
parser.add_option("-s",
"--host",
dest="host",
default="127.0.0.1",
help="Redis host name or address")
parser.add_option("-p",
"--port",
dest="port",
default=6379,
type="int",
help="Redis port")
parser.add_option("-d",
"--db",
dest="db",
default=0,
type="int",
help="Redis database")
parser.add_option("-a",
"--password",
dest="password",
default="",
help="Redis password")
parser.add_option("-l", "--log", dest="log", help="log file")
(options, args) = parser.parse_args(sys.argv)
if options.log:
create_log(options.log)
else:
create_log()
if len(args) > 1:
config = ConfigParser()
config.read(args[1])
guard = PushGuard(app_dir=config.get('app', 'app_dir'),
server_info={
'host': config.get('redis', 'host'),
'port': int(config.get('redis', 'port')),
'db': int(config.get('redis', 'db')),
'password': config.get('redis', 'password')
})
else:
guard = PushGuard(app_dir=options.app_dir,
server_info={
'host': options.host,
'port': options.port,
'db': options.db,
'password': options.password
})
guard.run()
def send_random_logs(arduino_monitor='localhost:8080', sleep_time=2):
for i in range(10):
random_value = random.uniform(0, 100)
mock_data = create_log(random_value, random_value, random_value,
random_value)
print('Sending mock data to {}... {}'.format(arduino_monitor,
(10 - i)))
send_log('http://{}/logs'.format(arduino_monitor), mock_data)
time.sleep(sleep_time)
def main():
query, num_start_pages, n, page_link_limit, mode, relevance_threshold = get_input(
)
start_time = time.time()
start_pages = get_start_pages(query, num_start_pages)
links_to_parse = PriorityQueue()
parsed_urls = ParsedURLs()
# get synonyms list and lemmatized words
synonyms, lemmatized_words = get_synonyms_and_lemmatized(query)
# creating a combined list of synonyms without duplicates
synonyms_list = list(
set([s for sublist in list(synonyms.values()) for s in sublist]))
print('Found %d crawlable start pages:\n' % len(start_pages))
# enqueue the start pages after computing their promises
for s in start_pages:
# promise = get_promise(query, s, mode, 0) # initially, parent_relevance is 0
promise = 1 # assuming that all the start pages are equally promising
links_to_parse.enqueue([promise, s])
# display the queue
links_to_parse.display_queue()
print('\n')
while links_to_parse and page_count.get_page_num() < n:
crawler = Crawler(links_to_parse, parsed_urls, query, n,
page_link_limit, mode, synonyms_list,
lemmatized_words)
crawler.run()
end_time = time.time()
total_time = (end_time - start_time) / 60 # minutes
# compute harvest rate
harvest_rate = get_harvest_rate(parsed_urls, relevance_threshold)
# create a crawler log file
create_log(parsed_urls, query, len(start_pages),
len(parsed_urls.get_keys()), page_link_limit, n, mode,
harvest_rate, relevance_threshold, total_time)
def execute():
parser = OptionParser(usage="%prog config [options]")
parser.add_option("-f", "--folder", dest="app_dir",
help="folder where the certs and keys to stay")
parser.add_option("-s", "--host", dest="host", default="127.0.0.1",
help="Redis host name or address")
parser.add_option("-p", "--port", dest="port", default=6379, type="int",
help="Redis port")
parser.add_option("-d", "--db", dest="db", default=0, type="int",
help="Redis database")
parser.add_option("-a", "--password", dest="password", default="",
help="Redis password")
parser.add_option("-l", "--log", dest="log",
help="log file")
(options, args) = parser.parse_args(sys.argv)
if options.log:
create_log(options.log)
else:
create_log()
if len(args) > 1:
config = ConfigParser()
config.read(args[1])
guard = PushGuard(app_dir=config.get('app', 'app_dir'),
server_info={'host': config.get('redis', 'host'),
'port': int(config.get('redis',
'port')),
'db': int(config.get('redis', 'db')),
'password': config.get('redis',
'password')})
else:
guard = PushGuard(app_dir=options.app_dir,
server_info={'host': options.host,
'port': options.port,
'db': options.db,
'password': options.password})
guard.run()
def main():
query, sql_password, user, password, selected_browser = get_args()
os.environ['sql_password'] = sql_password
# drop_database()
log = logger.create_log()
log.info('Starting to collect data')
print('Starting to collect data')
create_db.main()
soup_tweets = get_tweets(log=log,
query=query,
user=user,
password=password,
user_browser=selected_browser)
tweets_dict = create_tweets_obj(tweets=soup_tweets, log=log)
store_tweets_dict(tweets_dict, query, user, log)
log.info('Finished running')
print('Finished running')
CFG_FNAME = "./config/easyts.cfg"
# append the current path so the local packages can be imported
SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
sys.path.append(SCRIPT_PATH)
# import the local packages
from config import Config
import logger
# start the global 'log' and 'cfg' variables
cfg = Config(cfg_file=CFG_FNAME)
log = logger.create_log(tofile=cfg.log_file,
file_level=cfg.log_file_level,
file_name=cfg.log_file_name,
toconsole=cfg.log_console,
console_level=cfg.log_console_level,
tosyslog=cfg.log_syslog,
syslog_level=cfg.log_syslog_level,
syslog_address=cfg.log_syslog_address)
# log the entry call
log.debug("start LH Performance Data Service")
def err(msg):
sys.stderr.write("%s\n" % msg)
class FileLock(object):
def lock(self, fname, lkpath):
self._fname = fname
suite.addTests(test)
tests = dis.discover('envoyctl', 'test_proxy_config.py')
for test in tests:
suite.addTests(test)
tests = dis.discover('envoyctl', 'test_iptables.py')
for test in tests:
suite.addTests(test)
runner = xmlrunner.XMLTestRunner(output='test-reports')
runner.run(suite)
def set_env(env):
with open('update_env.txt', 'wr') as f:
f.writelines('env:%s' % env)
print("create update_env time is %s" % time.time())
@click.command()
@click.option('--feature', help='The feature you want to test.')
@click.option('--env', help='The environment you want to run tests')
def parse_command(feature, env):
print("feature is %s, env is %s" % (feature, env))
set_env(env)
find_testsuit(feature)
if __name__ == "__main__":
logger.create_log()
parse_command()
TYPE_PLAIN = "text/plain"
CFG_FNAME = "./config/easyts.cfg"
# append the current path so the local packages can be imported
SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
sys.path.append(SCRIPT_PATH)
# import the local packages
from config import Config
import logger
# start the global 'log' and 'cfg' variables
cfg = Config(cfg_file=CFG_FNAME)
log = logger.create_log(tofile=cfg.log_file, file_level=cfg.log_file_level, file_name=cfg.log_file_name,
toconsole=cfg.log_console, console_level=cfg.log_console_level,
tosyslog=cfg.log_syslog, syslog_level=cfg.log_syslog_level, syslog_address=cfg.log_syslog_address)
# log the entry call
log.debug("start LH Performance Data Service")
def err(msg):
sys.stderr.write("%s\n" % msg)
class FileLock(object):
def lock(self, fname, lkpath):
self._fname = fname