def _run_api_call(action, data=None):
'''
Runs a call against the SpiderOak API.
Returns a python object containing the response.
'''
log = logging.getLogger('run_api_call')
uri = API_URL_BASE % (get_config()['api_root'], action, )
https_handler = https.VerifiedHTTPSHandler()
https_opener = urllib2.build_opener(https_handler)
urllib2.install_opener(https_opener)
auth_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
auth_mgr.add_password(realm=None, uri=uri,
user=get_config()['api_user'],
passwd=get_config()['api_password'])
log.debug("Trying with user %s and pass %s" % (get_config()['api_user'],
get_config()['api_password'],))
auth_handler = urllib2.HTTPBasicAuthHandler(auth_mgr)
auth_opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(auth_opener)
if data is None:
fh = urllib2.urlopen(uri)
else:
datastr = json.dumps(data)
fh = urllib2.urlopen(uri, datastr)
json_string = fh.read()
retr_data = json.loads(json_string)
return retr_data
def read_students_list():
"""
Read students list from CSV file and return list with datamodel.Student
objects.
"""
students = []
csv.register_dialect('students', quoting=csv.QUOTE_NONE)
try:
with open(common.get_config().get_students_csv_file_path(),
encoding="u8",
newline='') as f:
reader = csv.reader(f, 'students')
for row in reader:
if len(row) != 2:
logger.print_error("incorrect students list CSV file")
return None
else:
student = datamodel.Student()
student.set_login(row[0])
student.set_name(row[1])
students += [student]
except IOError:
logger.print_error("cannot open students list " +
common.get_config().get_students_csv_file_path())
return None
if len(students) < 1:
logger.print_error("none students has been found")
else:
logger.print_msg("students list successfully loaded")
return students
def get_output(task, prog_path):
"""
Run program specified by prog_path and return tuple (output, error_output).
"""
# check task language and set relevant interpreter
if task.get_language() == "python2":
interpreter = common.get_config().get_python2path()
elif task.get_language() == "python3":
interpreter = common.get_config().get_python3path()
elif task.get_language() == "ruby":
interpreter = common.get_config().get_ruby_path()
else:
logger.print_error("unknown task language")
return ("", "")
# run subprocess
try:
p = subprocess.Popen([interpreter, prog_path],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except OSError:
logger.print_error("cannot run interpreter for language " + \
task.get_language())
return ("", "")
# send input to subprocess in another thread
outputs = ["", ""]
t = Thread(target=subprocess_communicate,
args=(p, task.get_input(), outputs))
t.start()
# wait until subprocess ends or time out
timeout = common.get_config().get_evaluator_timeout()
start = datetime.datetime.now()
while p.poll() == None:
sleep(0.1)
now = datetime.datetime.now()
if (now - start).seconds > timeout:
p.kill()
logger.print_warning("subprocess with interpreter " + \
task.get_language() + " " + \
"has been killed")
break
# wait until communication thread ends or time out (5 sec)
timeout = 5
start = datetime.datetime.now()
while t.is_alive():
sleep(0.1)
now = datetime.datetime.now()
if (now - start).seconds > timeout:
logger.print_warning("subprocess communication thread is " + \
"still running, interpreter: " + \
task.get_language())
break
return (outputs[0], outputs[1])
def __init__(self, origin_url):
'''
(CrawlerSource, str) -> CrawlerSource
creates a CrawlerSource with a given origin_url
'''
self.origin_url = origin_url
self.visit_queue = [origin_url]
self.visited_urls = []
self.domain = urlparse(origin_url).netloc
self.pages_visited = 0
self.probabilistic_n = common.get_config()["crawler"]["n"]
self.probabilistic_k = common.get_config()["crawler"]["k"]
def __init__(self, origin_url, filters):
'''
(Crawler, str) -> Crawler
creates a Crawler with a given origin_url
'''
self.origin_url = origin_url
self.filters = filters
self.visit_queue = collections.deque([origin_url])
self.visited_urls = set()
self.domain = urlparse(origin_url).netloc
self.pages_visited = 0
self.probabilistic_n = common.get_config()["crawler"]["n"]
self.probabilistic_k = common.get_config()["crawler"]["k"]
def setup_application():
config = get_config()
if config is not None:
return
config = read_config_file()
validate_config(config)
set_config(config)
def send_task(filename):
config_fn = DEFAULT_CONFIG_FN if not filename else filename
conf = get_config(config_fn)
client = scaleapi.ScaleClient(conf['test_api_key'])
try:
ret_obj = client.create_audiotranscription_task(
callback_url = conf.get('callback_url'),
attachment_type = conf.get('attachment_type'),
attachment = conf.get('attachment'),
verbatim = conf.get('verbatim')
)
except scaleapi.ScaleException as e:
sys.exit('\nStatus Code {}: {}\n\n(exited with error code 1)\n'.format(e.code, str(e)))
nod = lambda x: "Yes." if True else "No."
fmt_strs = (ret_obj.task_id,
ret_obj.created_at,
nod(ret_obj.is_test),
ret_obj.callback_url,
json.dumps(ret_obj.params, indent=4))
message = "Task (task_id={}) created at {}.\nWas this a test? {}\n\
Make sure this URL is listening for POST requests: {}\n\
Here are the parameters you used:\n{}".format(*fmt_strs)
create_log_filepath = get_log_filepath(conf) + 'task_{}_create.log'.format(ret_obj.created_at)
with open(create_log_filepath, 'w') as create_log_file:
create_log_file.write(message)
print("\n{}\n\n(A file containing this message has been written to `{}`.)\n"\
.format(message, create_log_filepath))
def auto_check():
try:
logging.info("auto check cpu_fs switch start!")
file = open("/sys/class/thermal/thermal_zone0/temp")
temp = float(file.read()) / 1000
file.close()
print "cpu temperature : %.1f" % temp
logging.info("cpu_fs check,temperature : %.1f" % temp)
#client auto task to check that wheather open the water switch
config_api_url = common.get_config("api_url")
#get the config info from server
api_url = config_api_url + "/data?type=config"
result = requests.get(url=api_url)
if result.status_code == 200:
result_text = result.text
logging.info("config info:" + result_text)
config_info = json.loads(result_text)
print config_info
config_cpu_temperature = config_info['data']['cpu_temperature']
#check
if temp > config_cpu_temperature:
return fs_open()
else:
return fs_close()
else:
raise Exception("can not get config info from the server!")
except Exception as e:
print "can not get config info from the server!"
logging.info(e)
def init_config(self):
config = {
"REFRESH_TOKEN": "refresh_token",
"DRIVE_ID": "drive_id",
"ROOT_PATH": "root",
"FILE_PATH": get_running_path(),
"MULTITHREADING": False,
"MAX_WORKERS": 5,
"CHUNK_SIZE": 104857600,
"RESUME": False,
"OVERWRITE": False,
"RETRY": 0,
"RESIDENT": False,
"VERSIONS": "v2021.0729.1800"
}
if not os.path.exists(get_config_file_path()):
self.print('请配置好config.json后重试', 'error')
with open(get_config_file_path(), 'w') as f:
f.write(json.dumps(config))
suicide()
try:
config.update(get_config())
DATA['config'] = config
except Exception as e:
self.print('请配置好config.json后重试', 'error')
raise e
def lambda_handler(event, context):
serial = ""
action = ""
res = None
if not event['queryStringParameters'].has_key('serial'):
return fatal_error("Firewall serial number not passed")
else:
serial = event['queryStringParameters']['serial']
if not event['queryStringParameters'].has_key('action'):
return fatal_error("Action (up or down) parameter not passed")
else:
action = event['queryStringParameters']['action']
config = get_config(CONFIG_BUCKET, CONFIG_FILE)
if not config['firewalls'].has_key(serial):
return fatal_error("Firewall serial number not found in configuration")
if action == "up":
res = up(config, serial)
elif action == "down":
res = down(config, serial)
else:
return fatal_error("Invalid action specified")
return res
def check_command():
""" (None) -> None
Check the communication file for any commands given.
Execute according to the commands.
"""
# Load the relevant configs
logging.debug("Checking for any new command on communication stream")
conf = common.get_config()['communication']
msg = comm_read()
# Let the output print back to normal for printing status
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
if msg[0] == 'W':
command = msg[1]
if command == 'S':
print('Stopping Explorer...')
logging.warning("Stop command detected, Stopping.")
comm_write('SS %s' % os.getpid())
sys.exit(0)
elif command == 'P':
print('Pausing ...')
logging.warning("Pause command detected, Pausing.")
comm_write('PP %s' % os.getpid())
while comm_read()[1] == 'P':
logging.info('Waiting %i seconds ...' % conf['sleep_time'])
print('Waiting %i seconds ...' % conf['sleep_time'])
time.sleep(conf['sleep_time'])
check_command()
elif command == 'R':
print('Resuming ...')
logging.warning('Resuming.')
comm_write('RR %s' % os.getpid())
def get_job_details(response):
logger.debug('Executing get_job_details')
try:
skip_user_query = "spark"
json_data_dump = capi.get_response_data(response)
if json_data_dump['apps'] is None: return None
data_arr = json_data_dump['apps']['app']
job_count = 0
data_list = list()
for key in data_arr:
logger.debug('complete application response %s: ' % key)
if key.get('elapsedTime') > int(capi.get_config('check_interval')):
logger.debug('A job is running more than an hour, fetching info. Elapsed time: {0}'.format(key.get('elapsedTime')))
job_count += 1
data = {"appId": key['id'], "allocatedMB": key['allocatedMB'], "user": key['user'], "jobCounts": job_count,
"trackingUrl": key['trackingUrl'], "startedTime": key['startedTime'], "elapsedTime": key['elapsedTime']};
app_landing_url = capi.get_base_url() + '/cluster/app/' + data.get('appId')
logger.debug('App landing url is: %s', app_landing_url)
data["app_landing_url"] = app_landing_url
# if skip_user_query not in key.get('name').lower():
# user_query = get_user_query(key.get('trackingUrl', ''))
# if user_query == '' or user_query is None:
# user_query = key['name']
# data["user_query"] = user_query
data_list.append(data)
return data_list
except Exception as e:
logger.error('Exception from get_job_details'.format(e.message))
raise e
def auto_check():
try:
logging.info("auto check water switch start!")
#client auto task to check that wheather open the water switch
config_api_url = common.get_config("api_url")
#get the config info from server
api_url = config_api_url + "/data?type=config"
result = requests.get(url=api_url)
if result.status_code == 200:
result_text = result.text
logging.info("config info:" + result_text)
config_info = json.loads(result_text)
print config_info
config_temperature = config_info['data']['temperature']
config_humidity = config_info['data']['humidity']
#check
room_info = temperature.get_info()
room_temperature = room_info[0]
room_humidity = room_info[1]
is_wrong = room_info[2]
#if room_temperature > config_temperature and is_wrong == False:
if room_temperature > config_temperature and room_temperature < 50:
return open()
if room_humidity < config_humidity:
return open()
return close()
else:
raise Exception("can not get config info from the server!")
except Exception as e:
print "can not get config info from the server!"
logging.info(e)
def initialize_secdb():
'''
initialize secdbmodule
:param params: a dic with following keys:
secdb_url => connect string to the securities database
parallelize => a boolean that indicate (only needed if you use future rolling feature)
workers => number of processes for parallelization
'''
# invalidate previous session if it existed
global _session
global __params
if _session is not None:
assert isinstance(_session,scoped_session)
_session.remove()
_session = None
params = dict(get_config().items("mysql"))
assert isinstance(params, dict)
# assert 'parallelize' in params # excluded: not needed inside the package
# assert 'workers' in params # excluded: not needed inside the package
__params = params
def main(files, sql, onlyusers, upload, delete):
auth_data = get_config()
db, con = mysql_connect(auth_data)
accs = get_hosts(con)
folder = auth_data['backup_dir'] + str(datetime.date.today()) + '/'
for acc in accs:
if onlyusers and (acc['name'] not in onlyusers):
print "Ignoring %s" % acc['name']
continue
if not upload:
folder = acc['root'] + "/backup/"
mkdir_recursive(folder)
if files:
print "Making %s's files backup" % acc['name']
filename = files_dump(auth_data, acc, folder)
add_backup_record(con, acc, filename, upload, folder, "files")
if sql:
print "Making %s's DB backup" % acc['name']
filename = mysql_dump(auth_data, acc, folder)
add_backup_record(con, acc, filename, upload, folder, "sql")
db.commit()
if upload:
print "uploading"
upload_to_ftp(auth_data, con, db)
if delete:
print "Removing backup"
print "[removing "+folder+"]"
#os.removedirs(folder)
else:
print "Keeping backup"
db.close()
def main():
"""
Connect to the specified device so that the tester can send commands to it.
"""
parser = argparse.ArgumentParser()
parser.add_argument('Device', type=str, help="Specify which device (either" \
"valve or mcpc) to connect to.")
cfg = get_config()
# Parse the input argument to determine which device to connect to.
args = parser.parse_args()
device = None
if args.Device == "valve":
device = ThreeWayValve()
device.connect(port=cfg.valve_port,
baudrate=cfg.valve_baud,
reset=False)
elif args.Device == "mcpc":
device = MCPC()
device.connect(port=cfg.mcpc_port, baudrate=cfg.mcpc_baud)
else:
print("Please specify whether to connect to the valve or mcpc.")
return
# Set up a command line interface.
try:
while True:
cmd = input('cmd: ')
device._write(cmd + '\n')
time.sleep(0.1)
print('\n'.join(
map(repr,
device.read_all().splitlines(keepends=True))))
except (KeyboardInterrupt, EOFError):
pass
def _validate_provider(provider, **kwargs):
"""
Validate the feeds for a provider.
"""
# compute a time query range; one or both sides may not be relevant for all feeds.
if "start_time" not in kwargs and "end_time" not in kwargs:
# default to the hour beginning 25 hours before the current time
end = datetime.datetime.utcnow() - datetime.timedelta(days=1)
start = end - datetime.timedelta(seconds=3600)
elif "start_time" not in kwargs or "end_time" not in kwargs:
# one side of range provided, compute the other side for a total range of an hour
start, end = common.parse_time_range(duration=3600, **kwargs)
else:
# both sides of range provided
start, end = common.parse_time_range(**kwargs)
kwargs["start_time"] = start
kwargs["end_time"] = end
config = common.get_config(provider, kwargs.get("config"))
# assert the version parameter
version = mds.Version(config.pop("version", kwargs.get("version")))
version.raise_if_unsupported()
kwargs["version"] = version
kwargs["no_paging"] = False
kwargs["rate_limit"] = 0
kwargs["client"] = mds.Client(provider, version=version, **config)
return _validate(**kwargs)
def setup_logging(site_name="", increment=True):
# Load the relevant configs
config = common.get_config()
# Logging config
current_time = datetime.datetime.now().strftime('%Y%m%d')
log_dir = config['projectdir']+"/log"
prefix = log_dir + "/" + site_name + "article_explorer-"
try:
cycle_number = sorted(glob.glob(prefix + current_time + "*.log"))[-1][-7:-4]
if increment:
cycle_number = str(int(cycle_number) + 1)
except (KeyboardInterrupt, SystemExit):
raise
except:
cycle_number = "0"
# Remove all handlers associated with the root logger object.
# This will allow logging per site
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
logging.basicConfig(filename=prefix + current_time + "-" + cycle_number.zfill(3) + ".log",
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
default_logger = logging.getLogger('')
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.WARNING)
console_handler.setFormatter(default_logger.handlers[0].formatter)
default_logger.addHandler(console_handler)
def ok(self):
_range = self.settings_dict['range']
option = self.settings_dict['option']
if _range == 'rng1': # Ask everytime
transfer = self.transfers_list[self.current_index]
self.skip_or_restart(option, transfer)
elif _range == 'rng2': # Apply only for this session'
if self.type == 'upload':
self.manager.set_transfer_settings(uploads=option)
else:
self.manager.set_transfer_settings(downloads=option)
self.set_for_all_transfers(option)
self.popup.dismiss()
elif _range == 'rng3': # 'Set as default':
config = get_config()
# noinspection PyBroadException
try:
config.add_section('DEFAULTS')
except Exception:
pass
config.set('DEFAULTS', self.type, option)
with open(config_file, 'w+') as f:
config.write(f)
self.manager.get_default_settings()
self.set_for_all_transfers(option)
self.popup.dismiss()
self.manager.start_transfers()
if len(self.transfers_list) == 0:
self.dismiss()
def getWarc(request, filename):
config = common.get_config()['warc']
path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../../',
config['dir'] + "/" + config['twitter_subdir']))
filename_ext = path + "/" + filename + ".warc.gz"
warc = open(filename_ext, "rb")
res = HttpResponse(warc, content_type="application/force-download")
warc.close()
# To inspect details for the below code, see http://greenbytes.de/tech/tc2231/
if u'WebKit' in request.META['HTTP_USER_AGENT']:
# Safari 3.0 and Chrome 2.0 accepts UTF-8 encoded string directly.
filename_header = 'filename=%s' % (filename +
'.warc.gz').encode('utf-8')
elif u'MSIE' in request.META['HTTP_USER_AGENT']:
# IE does not support internationalized filename at all.
# It can only recognize internationalized URL, so we do the trick via routing rules.
filename_header = ''
else:
# For others like Firefox, we follow RFC2231 (encoding extension in HTTP headers).
filename_header = 'filename*=UTF-8\'\'%s' % urllib.quote(
(filename + '.warc.gz').encode('utf-8'))
res['Content-Disposition'] = 'attachment; ' + filename_header
return res
def parse_all_tasks():
"""
Parse all tasks from user specified folders.
Return list with tuples (tasks, number) where tasks is list of
datamodel.Task objects and number is number of tasks per student from
this list.
"""
tasks = []
tasks_paths = common.get_config().get_tasks_paths()
for (task_path, points, number) in tasks_paths:
tasks_from_folder = parse_tasks(task_path)
# something got wrong
if tasks_from_folder == None:
logger.print_error("there is something wrong with folder " +
task_path)
return None
# too little tasks in folder
if len(tasks_from_folder) < number:
logger.print_error("too little tasks in folder " +
task_path + ", need " + str(number))
return None
# set max points for every task in folder
for task in tasks_from_folder:
task.set_max_points(points)
# add tuple (tasks_from_folder, number) to the end list
tasks += [(tasks_from_folder, number)]
return tasks
def serve(students, tasks):
"""
Assign tasks to students (if anonymous mode is off) and start web server.
"""
# init global variables
global _tasks
global _students
global _templates
global _status
_tasks = tasks
_templates = load_templates()
# assign tasks to students
if not common.get_config().is_anonymous():
for student in students:
logger.print_msg("assigning tasks for student: " + \
student.get_login())
tasks_for_student = util.random_assign(tasks)
# generate random session id
m = hashlib.md5()
m.update((student.get_name() + \
student.get_login() + \
str(random.randrange(0xFFFFFFFFFFFFFFFF))).encode("u8"))
sid = m.hexdigest()
_students += [(student, tasks_for_student, 0, sid)]
# start web service
try:
serve_forever()
except KeyboardInterrupt:
_status = 0
return _status
def setup_application():
config = get_config()
if config is not None:
return
config = read_config_file()
validate_config(config)
set_config(config)
def upload():
try:
# CPU informatiom
CPU_temp = getCPUtemperature()
CPU_usage = getCPUuse()
# RAM information
# Output is in kb, here I convert it in Mb for readability
RAM_stats = getRAMinfo()
RAM_total = round(int(RAM_stats[0]) / 1000, 1)
RAM_used = round(int(RAM_stats[1]) / 1000, 1)
RAM_free = round(int(RAM_stats[2]) / 1000, 1)
# Disk information
DISK_stats = getDiskSpace()
DISK_total = DISK_stats[0]
DISK_used = DISK_stats[1]
#DISK_perc = DISK_stats[3]
DISK_perc = DISK_stats[2]
config_api_url = common.get_config('api_url')
api_url = config_api_url + "/upload?type=cpu_info&temperature=%.2f&cpu_use=%.2f&ram_used=%s&ram_free=%s&disk_used=%s&disk_perc=%s" % (
float(CPU_temp), float(CPU_usage), str(RAM_used), str(RAM_free),
str(DISK_used), str(DISK_perc))
print api_url
#os.system("curl "+api_url)
result = requests.get(url=api_url)
logging.info('UPLOAD CPU INFO:' + api_url)
print "UPLOAD CPU INFO SUCCESS!"
except Exception as e:
print "UPLOAD CPU INFO SUCCESS failed!"
logging.info(e)
def getJson(request):
scope = {
'name': 'mediacat-scope',
'version': common.get_config()['database']['version'],
'date': time.strftime("%c"),
}
json_serializer = serializers.get_serializer('json')()
scope['referring_sites'] = json.loads(
json_serializer.serialize(ReferringSite.objects.all()))
scope['referring_sites_filter'] = json.loads(
json_serializer.serialize(ReferringSiteFilter.objects.all()))
scope['referring_sites_css_selector'] = json.loads(
json_serializer.serialize(ReferringSiteCssSelector.objects.all()))
scope['referring_twitter'] = json.loads(
json_serializer.serialize(ReferringTwitter.objects.all()))
scope['source_twitter'] = json.loads(
json_serializer.serialize(SourceTwitter.objects.all()))
scope['source_twitter_alias'] = json.loads(
json_serializer.serialize(SourceTwitterAlias.objects.all()))
scope['source_sites'] = json.loads(
json_serializer.serialize(SourceSite.objects.all()))
scope['source_sites_alias'] = json.loads(
json_serializer.serialize(SourceSiteAlias.objects.all()))
scope['keywords'] = json.loads(
json_serializer.serialize(Keyword.objects.all()))
scope['taggit'] = json.loads(
json_serializer.serialize(TaggedItem.objects.all()))
res = HttpResponse(json.dumps(scope, indent=2, sort_keys=True))
res['Content-Disposition'] = format('attachment; filename=scope-%s.json' %
time.strftime("%Y%m%d-%H%M%S"))
return res
def run_ansible(config_dir, mflag=0, tags=""):
ret = os.system("python utils/gen_inventory.py")
if ret != 0:
print("ERROR: Error creating Ansible inventory")
return False
print("Success: hosts file created")
ansible_inventory = os.path.join(config_dir, 'hosts')
extra_vars = {}
public_ip = common.get_publicip()
extra_vars['public_ip'] = public_ip
extra_vars['public_ip_httpd'] = public_ip.replace('.', '\.')
extra_vars['mflag'] = mflag
extra_vars.update(common.get_config('webapp_repo'))
cmd = "ansible-playbook --extra-vars '" + str(
json.dumps(extra_vars)) + "' -i " + ansible_inventory + " play.yml"
if tags:
cmd += ' --tags "' + tags + '"'
ret = os.system(cmd)
if ret != 0:
print("ERROR: Error running Ansible playbook")
return False
return True
def run():
settings = get_config()
app.run(debug=debug,
host='0.0.0.0',
port=int(settings["webserver"]["port"]),
threaded=True)
return
def getWarc(request, filename):
try:
config = common.get_config()['warc']
path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../', config['dir'] + "/" + config['article_subdir']))
filename_ext = path + "/" + filename + ".warc.gz"
warc = open(filename_ext, "rb")
res = HttpResponse(warc, content_type="application/force-download")
warc.close()
# To inspect details for the below code, see http://greenbytes.de/tech/tc2231/
if u'WebKit' in request.META['HTTP_USER_AGENT']:
# Safari 3.0 and Chrome 2.0 accepts UTF-8 encoded string directly.
filename_header = 'filename=%s' % (filename + ".warc.gz").encode('utf-8')
elif u'MSIE' in request.META['HTTP_USER_AGENT']:
# IE does not support internationalized filename at all.
# It can only recognize internationalized URL, so we do the trick via routing rules.
filename_header = ''
else:
# For others like Firefox, we follow RFC2231 (encoding extension in HTTP headers).
filename_header = 'filename*=UTF-8\'\'%s' % urllib.quote((filename + ".warc.gz").encode('utf-8'))
res['Content-Disposition'] = 'attachment; ' + filename_header
return res
except IOError:
# if the warc file doesn't exist, display 404 page to user, and enqueue the url
# to task queue to retry generating warc
url = filename.replace("_", "/")
warc_creator.enqueue_article(url)
return render(request, '404/404.html')
def nmap_engine(ip_seg):
# 获取nmap输出路径
output_dir = os.path.join(sys.path[0], 'nmap_output')
if not os.path.exists(output_dir):
os.mkdir(output_dir)
# 获取nmap输出文件名
output_file = os.path.join(output_dir, time.strftime("%Y%m%d%H%m%S"))
# 读取配置中nmap的参数并调用masscan
nmap_max_rate = get_config('scan', 'max_rate')
nmap_port_range = get_config('scan', 'port_range')
res = my_nmap.run(ip_seg=ip_seg,
output_file=output_file,
rate=nmap_max_rate,
port_range=nmap_port_range)
return res
def __init__(self, logger_name="example"):
# create logger
self.logger_name = logger_name
self.logger = logging.getLogger(self.logger_name)
self.logger.setLevel(logging.DEBUG)
# create file handler
self.log_base_path = './log'
self.log_path = "{}/{}.log".format(self.log_base_path,
self.logger_name)
if not os.path.exists(self.log_base_path):
os.mkdir(self.log_base_path)
self.fh = logging.FileHandler(self.log_path, encoding='utf8')
fh_level = get_config('conf.ini', 'LOG', 'level')
self.fh.setLevel(logging.getLevelName(fh_level))
# create stream handler
self.sh = logging.StreamHandler()
self.sh.setLevel(logging.DEBUG)
# create formatter
self.fmt = "[%(asctime)s][%(levelname)s][%(name)s][%(process)d][%(filename)s][%(funcName)s][line:%(lineno)d] %(message)s"
self.datefmt = "%Y-%m-%d %H:%M:%S"
self.formatter = logging.Formatter(self.fmt, self.datefmt)
# add handler and formatter to logger
self.fh.setFormatter(self.formatter)
self.sh.setFormatter(self.formatter)
self.logger.addHandler(self.fh)
self.logger.addHandler(self.sh)
def populate_bigquery_table():
"""Create & populate input table based on what is configured for event input in klio-job.yaml
This needs to run before `klio job run` is called, which is why
"""
table_schema = {
"fields": [{
'name': 'entity_id',
'type': 'STRING',
'mode': 'NULLABLE'
}]
}
klio_cfg = common.get_config()
input_table_cfg = klio_cfg.job_config.events.inputs[0]
table_name = "{}:{}.{}".format(input_table_cfg.project,
input_table_cfg.dataset,
input_table_cfg.table)
with beam.Pipeline() as p:
def create_record(v):
return {
'entity_id': v,
}
record_ids = p | 'CreateIDs' >> beam.Create(common.entity_ids)
records = record_ids | 'CreateRecords' >> beam.Map(
lambda x: create_record(x))
records | 'write' >> beam.io.WriteToBigQuery(
table_name,
schema=table_schema,
create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE)
def eval_fulltext(student, task, num, formdata, points):
"""
Evaluate student's answer on fulltext task.
"""
# write task to the output folder
path = os.path.join(common.get_config().get_output_path(), \
student.get_login() + "_task_" + \
'{:0>2}'.format(str(num+1)) + ".txt")
try:
answer = formdata["a_" + str(num)][0]
except (KeyError, IndexError):
answer = ""
if common.get_config().get_evaluator_strictness() == 0:
pattern = re.compile(r'\s+')
cleaned_answer = re.sub(pattern, '', answer)
else:
cleaned_answer = answer.strip()
final_points = 0
for correct_answer, coef in task.get_answers():
if common.get_config().get_evaluator_strictness() == 0:
pattern = re.compile(r'\s+')
correct_answer = re.sub(pattern, '', correct_answer)
else:
correct_answer = correct_answer.strip()
if string_equals(cleaned_answer, correct_answer):
p = task.get_max_points() * coef
if p.is_integer():
p = int(p)
else:
p = 0
final_points = max(final_points, p)
try:
with open(path, "w", encoding="u8") as f:
print(QUESTION, file=f)
print(task.get_question().strip(), file=f)
for correct_answer, coef in task.get_answers():
print(CORRECT_ANSWER, file=f)
print(correct_answer.strip() + " (" + str(coef) + ")", file=f)
print(ANSWER, file=f)
print(answer.strip(), file=f)
print(POINTS, file=f)
print(final_points, file=f)
except IOError:
logger.print_error("cannot write file to the output folder")
# set points
points[num] = final_points
def write_points(student, points):
path = os.path.join(common.get_config().get_output_path(), \
student.get_login() + "_points.txt")
try:
with open(path, "w", encoding="u8") as f:
print(str(round(sum(points), 2)), file=f)
except IOError:
logger.print_error("cannot write file to the output folder")
def main():
setup_logging('ddns2.log')
config = get_config('router', 'ddns')
logging.info("starting ddns")
ddns_config = config['DDNS_CONFIG']
resource4 = ddns_config['resource4']
resource6 = ddns_config['resource6']
domain = ddns_config['domain']
pppoe_interface = ddns_config['pppoe_interface']
ipv6_pool_name = ddns_config['ipv6_pool_name']
logging.info("loaded config, connecting to router")
router_api = RouterOsApiPool(**(config['ROUTER_CONFIG'])).get_api()
while True:
try:
for local_ip, resource in ((get_local_ipv4(router_api,
pppoe_interface),
resource4),
(get_local_ipv6(router_api,
ipv6_pool_name),
resource6)):
current_ip, record_name = get_current_record_ip(
domain, resource)
if local_ip is not None and current_ip != local_ip:
logging.info(
f"updating record {record_name} from {current_ip} to {local_ip}"
)
update_record_ip(domain, resource, local_ip)
heartbeat()
except RouterOsApiConnectionError:
logging.warning('resetting router api')
router_api = RouterOsApiPool(
**(get_config('router')['ROUTER_CONFIG'])).get_api()
except JSONDecodeError:
logging.exception('linode JSONDecodeError')
except (requests.exceptions.RequestException, OSError):
logging.exception('RequestException issues')
except:
logging.exception("error")
raise
finally:
sleep(30)
def runCfg():
cfg = common.get_config()
cfg['untouchedPeriod'] = common.util_convert_days_to_datetime(untouchedPeriodDays)
cfg['needInfoPingPeriod'] = common.util_convert_days_to_datetime(needInfoPingPeriodDays)
cfg['backportRequestPeriod'] = common.util_convert_days_to_datetime(backportRequestPeriodDays)
cfg['needInfoFollowUpPingPeriod'] = common.util_convert_days_to_datetime(needInfoFollowUpPingPeriodDays)
cfg['needsCommentPeriod'] = common.util_convert_days_to_datetime(needsCommentPeriodDays)
return cfg
def serve_forever():
"""
Start web server on user specified port
"""
server_address = ('', common.get_config().get_webserver_port())
httpd = http.server.HTTPServer(server_address, HTTPRequestHandler)
logger.print_msg("starting web service on port " + str(server_address[1]))
print_students_info()
httpd.serve_forever()
def getDump(request):
version = common.get_config()['database']['version']
out = StringIO()
management.call_command('dumpdata', 'explorer', 'taggit', stdout=out)
res = HttpResponse(version + '\n' + out.getvalue())
out.close()
res['Content-Disposition'] = format('attachment; filename=scope-%s.json' %
time.strftime("%Y%m%d-%H%M%S"))
return res
def check_password(self, formdata):
"""Return True is there is a correct password in form data"""
config = common.get_config()
try:
if formdata["password"][0] == config.get_webserver_pass():
return True
else:
return False
except (KeyError, IndexError):
return False
def create_article_pdf(url, file_name):
"""(url)-->None
giving url it will export pdf file
create_article_pdf("http://www.facebook.com")
it should have a .pdf file under
ARTICLE_WARC_DIR/http:__www.facebook.com.pdf
"""
config = common.get_config()["pdf"]
return create_pdf(url, file_name, config['dir'] + "/" + config['article_subdir'])
def authorize():
""" (None) -> tweepy.API
Will use global keys to allow use of API
"""
#Get's config settings for twitter
config = common.get_config()['twitter']
#Authorizing use with twitter development api
auth = tweepy.OAuthHandler(config['consumer_key'], config['consumer_secret'])
auth.set_access_token(config['access_token'], config['access_token_secret'])
return tweepy.API(auth)
def __init__(self):
self.logger = get_logger(__name__)
self.config = get_config()
self.bce_access_key_id = self.config['BCE_ACCESS_KEY_ID']
self.bce_secret_access_key = self.config['BCE_SECRET_ACCESS_KEY']
self.bce_bos_host = self.config['BCE_BOS_HOST']
self.bce_sts_host = self.config['BCE_STS_HOST']
self.bos_src_bucket = self.config['BOS_SRC_BUCKET']
self.bos_storage_class = self.config['BOS_STORAGE_CLASS']
self.bos_des_dir = self.config['BCE_SECRET_ACCESS_KEY']
def save(self):
config = get_config()
if config:
config.remove_section(self.section)
with open(config_file, 'w+') as f:
config.write(f)
for association_row in self.ids.associations_space.children:
association = association_row.get_association()
if association:
self.add_association(association_row)
def run_app(filename):
config_fn = DEFAULT_CONFIG_FN if not filename else filename
conf = get_config(config_fn)
app.config.update(
LIVE_CALLBACK_AUTH_KEY=conf.get('test_callback_auth_key'),
TEST_CALLBACK_AUTH_KEY=conf.get('live_callback_auth_key'),
datastores=conf.get('datastores'))
add_callback_rules(conf)
app.run()
def prepare_test_response(self, formdata):
global _students
# get tasks and student's session id
if common.get_config().is_anonymous():
# create new anonymous student
student = datamodel.Student()
login = "******" + '{:0>4}'.format(str(len(_students)))
student.set_login(login)
logger.print_msg("assigning tasks for student: " + login)
tasks = util.random_assign(_tasks)
# generate random session id
m = hashlib.md5()
m.update((login + \
str(random.randrange(0xFFFFFFFFFFFFFFFF))).encode("u8"))
sid = m.hexdigest()
_students += [(student, tasks, 1, sid)]
else:
login = formdata["login"][0]
i = 0
for (student_, tasks_, status, sid_) in _students:
if student_.get_login() == login:
student = student_
tasks = tasks_
sid = sid_
break
i += 1
# set correct status (1 = test already requested)
_students[i] = (student, tasks, 1, sid)
# create xhtml form
inputsdata = ""
if not common.get_config().is_anonymous():
inputsdata += "<i>" + student.get_login() + ", " + \
student.get_name() + "</i><br/><br/>"
i = 0
for task in tasks:
inputsdata += self.create_inputs(task, i)
i += 1
inputsdata += '<input type="hidden" value="' + sid + '" name="' + \
'sid"/>'
# return created form
resp = _templates[1].replace(r"#replaceinputs#", inputsdata)
return resp.encode("u8")
def __init__(self):
super(EasyWindow, self).__init__()
self.setWindowTitle('快捷使用')
self.tabWidget = QtWidgets.QTabWidget()
self.setCentralWidget(self.tabWidget)
self._widgets = []
players = common.get_config(config.conf_file, 'players')
for player_id in players.keys():
widget = OptionWidget(players[player_id]['address'])
self.tabWidget.addTab(widget, '{}号机器人'.format(int(player_id)))
self._widgets.append(widget)
def create_article_pdf(url, file_name):
"""(url)-->None
giving url it will export pdf file
create_article_pdf("http://www.facebook.com")
it should have a .pdf file under
ARTICLE_WARC_DIR/http:__www.facebook.com.pdf
"""
config = common.get_config()["pdf"]
return create_pdf(url, file_name,
config['dir'] + "/" + config['article_subdir'])
def save_latex_file(latex_str, id_):
"""
Save latex file to the output folder.
"""
path = os.path.join(common.get_config().get_output_path(),
'{:0>4}'.format(str(id_)) + ".tex")
try:
with open(path, "w", encoding="u8") as f:
print(latex_str, file=f)
except IOError:
logger.print_error("cannot write file to the output folder")
def create_twitter_warc(url):
"""(url)-->None
giving url it will export warc file
create_warc("https://twitter.com/LeagueOfLegends")
it should have a HTML file under
TWITTER_WARC_DIR/https:__twitter.com_LeagueOfLegends
"""
file_name = url.replace("/", "_")
config = common.get_config()["warc"]
return create_warc(url, file_name, config['dir'] + "/" + config['twitter_subdir'])
def on_upload_setting(self, *_):
config = get_config()
# noinspection PyBroadException
try:
config.add_section('DEFAULTS')
except Exception:
pass
config.set('DEFAULTS', 'upload', self.tsm._option(self.upload_setting))
with open(config_file, 'w+') as f:
config.write(f)
def config_file_check():
try:
config = get_config(USER_CONFIG_PATH)
if not (config['DEFAULT']['weather_location']
and config['DEFAULT']['weather_api_key']
and config['DEFAULT']['weather_units']):
return False
return config['DEFAULT']['weather_location'], config['DEFAULT'][
'weather_api_key'], config['DEFAULT']['weather_units']
except KeyError:
return False
def delete_tables():
klio_cfg = common.get_config()
input_table_cfg = klio_cfg.job_config.events.inputs[0]
output_table_cfg = klio_cfg.job_config.events.outputs[0]
bq_client = beam_bq_tools.BigQueryWrapper()
bq_client._delete_table(input_table_cfg.project, input_table_cfg.dataset,
input_table_cfg.table)
bq_client._delete_table(output_table_cfg.project, output_table_cfg.dataset,
output_table_cfg.table)
def main():
# create experiment config containing all hyperparameters
config = get_config('train')
# create network and training agent
tr_agent = get_agent(config)
# load from checkpoint if provided
if config.cont:
tr_agent.load_ckpt(config.ckpt)
# create dataloader
train_loader = get_dataloader('train', config)
val_loader = get_dataloader('validation', config)
val_loader = cycle(val_loader)
# start training
clock = tr_agent.clock
for e in range(clock.epoch, config.n_epochs):
# begin iteration
pbar = tqdm(train_loader)
for b, data in enumerate(pbar):
# train step
outputs, losses = tr_agent.train_func(data)
# visualize
if config.vis_frequency is not None and clock.step % config.vis_frequency == 0:
tr_agent.visualize_batch(data, "train", outputs=outputs)
pbar.set_description("EPOCH[{}][{}]".format(e, b))
pbar.set_postfix(
OrderedDict({k: v.item()
for k, v in losses.items()}))
# validation step
if clock.step % config.val_frequency == 0:
data = next(val_loader)
outputs, losses = tr_agent.val_func(data)
if config.vis_frequency is not None and clock.step % config.vis_frequency == 0:
tr_agent.visualize_batch(data,
"validation",
outputs=outputs)
clock.tick()
tr_agent.update_learning_rate()
clock.tock()
if clock.epoch % config.save_frequency == 0:
tr_agent.save_ckpt()
tr_agent.save_ckpt('latest')
def prepare_index_response(self):
resp = re.sub(r"#replacepass#([^#]+)#replacepass#",
r'\1<input type="password" name="password" />',
_templates[0])
if common.get_config().is_anonymous():
resp = re.sub("#replacelogin#([^#]+)#replacelogin#",
'',
resp)
else:
resp = re.sub(r"#replacelogin#([^#]+)#replacelogin#",
r'\1<input type="text" name="login" />',
resp)
return resp.encode("u8")
def generate(self):
params = []
# init params by random numbers
for r in self.params_range:
params += [random.randrange(r)]
# generate python code and correct output
(code, output) = self.get_code_and_output(params)
task = ChoiceTask()
# create question
question = code.strip()
if common.get_config().get_mode() == "web":
question = "<pre>" + question + "</pre>"
elif common.get_config().get_mode() == "print":
question = "\n#BACKSLASH#texttt#LEFT_BRACE#" + question + \
"#RIGHT_BRACE#"
question = self.question + question
task.set_question(question)
# create answers
answers = [(output.strip(), True)]
while len(answers) < self.answers_num:
answer_generated = False
while not answer_generated:
iparam = random.randrange(len(self.params_range))
# try change one param... we need different, but similar output
params[iparam] = random.randrange(self.params_range[iparam])
(code, output) = self.get_code_and_output(params)
# check if output differs from standing answers
answer_generated = True
for (answer, rest) in answers:
if answer == output.strip():
answer_generated = False
break
if answer_generated:
answers += [(output.strip(), False)]
task.set_answers(answers)
return task
def getImg(request, filename):
try:
config = common.get_config()['pdf']
path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../', config['dir'] + "/" + config['article_subdir']))
filename_ext = path + "/" + filename + ".png"
pdf = open(filename_ext, "rb")
# open img in browser instead of downloading it
res = HttpResponse(pdf, content_type="image/png")
pdf.close()
return res
except IOError:
# if the img file doesn't exist, display 404 page to user, and enqueue the url
# to task queue to retry generating img
url = filename.replace("_", "/")
warc_creator.enqueue_article(url)
return render(request, '404/404.html')
def check_login(self, formdata):
"""
Return True is there is a student with given login and status 0 or 1
In anonymous mode this is always true.
"""
if common.get_config().is_anonymous():
return True
try:
login = formdata["login"][0]
for (student, tasks, status, sid) in _students:
if student.get_login() == login:
if status < 2:
return True
else:
return False
return False
except (KeyError, IndexError):
return False
def comm_read():
""" (None) -> Str
Reads the current status or command listed on the comm_file with the
article, then returns the output.
"""
# Load the relevant configs
conf = common.get_config()['communication']
# Wait for retry_count * retry_delta seconds
for k in range(conf['retry_count']):
try:
comm = open('article' + conf['comm_file'], 'r')
msg = comm.read()
comm.close()
return msg
except (KeyboardInterrupt, SystemExit):
raise
except:
time.sleep(conf['retry_delta'])
def download_options(self, obj):
root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../'))
config = common.get_config()
warc_file = root_dir + "/" + config['warc']['dir'] + "/" + config['warc']['article_subdir'] + "/" + obj.text_hash + ".warc.gz"
warc_available = os.path.isfile(warc_file)
pdf_file = root_dir + "/" + config['pdf']['dir'] + "/" + config['pdf']['article_subdir'] + "/" + obj.text_hash + ".pdf"
pdf_available = os.path.isfile(pdf_file)
img_file = root_dir + "/" + config['pdf']['dir'] + "/" + config['pdf']['article_subdir'] + "/" + obj.text_hash + ".png"
img_available = os.path.isfile(img_file)
# return format((
# """
# <div class="btn-group"> \
# <button class="btn btn-default" %s><a href="/articles/warc/%s">Download Warc</a></button> \
# <button class="btn btn-default" %s><a href="/articles/pdf/%s">View PDF</a></button> \
# <button class="btn btn-default" %s><a href="/articles/img/%s">View Screenshot</a></button> \
# </div> \
# """) %
# (
# '' if warc_available else 'disabled', obj.text_hash,
# '' if pdf_available else 'disabled', obj.text_hash,
# '' if img_available else 'disabled', obj.text_hash,
# ))
return format((
"""
<div class="btn-group"> \
<a class="btn btn-success %s" %s>Download Warc</a> \
<a class="btn btn-success %s" target="_blank" %s>View PDF</a> \
<a class="btn btn-success %s" target="_blank" %s>View Screenshot</a> \
</div> \
""") %
(
'' if warc_available else 'disabled', 'href="/articles/warc/' + obj.text_hash + '"' if warc_available else '',
'' if pdf_available else 'disabled', 'href="/articles/pdf/' + obj.text_hash + '"' if pdf_available else '',
'' if img_available else 'disabled', 'href="/articles/img/' + obj.text_hash + '"' if img_available else '',
))
def authenticate_user(environ, start_response):
log = logging.getLogger('authenticate_user')
log.debug("start")
try:
brand_identifier = environ['query_data']['brand_id'][0]
username = environ['query_data']['username'][0]
password = environ['query_data']['password'][0]
crypt_pw = environ['query_data'].get('crypt_pw', ["True"])[0]
except KeyError:
log.error("Got bad request.")
return BadRequest()(environ, start_response)
decoded_user = unquote(username)
# If we get anything OTHER than explicitly "False" in the request, we will assume it's an encrypted password.
if crypt_pw == "False":
plaintext_password = password
else:
try:
plaintext_password = server.read_escrow_data(
brand_identifier, password)
except KeyError:
log.warn("missing identifier %s" % (brand_identifier,))
return NotFound()(environ, start_response)
except ValueError:
log.warn("bad values for authenticating user %s" % (decoded_user,))
return BadRequest()(environ, start_response)
except Exception:
log.exception("server.read_escrow_data failed for user %s brand %s"
% (decoded_user, brand_identifier,))
return ServerError()(environ, start_response)
if not authenticator(get_config(), decoded_user, plaintext_password):
log.info("Auth failed for %s" % (decoded_user,))
return Forbidden()(environ, start_response)
log.info("Auth OK for brand %s with user %s" % (brand_identifier, decoded_user, ))
return SuperSimple("OK")(environ, start_response)
def comm_write(text):
""" (Str) -> None
Writes a command to the comm_file of article.
The file is used to communicate with the running explorer process,
to change the status safely.
Keyword arguments:
text -- String of command
"""
# Load the relevant configs
conf = common.get_config()['communication']
# Wait for retry_count * retry_delta seconds
for k in range(conf['retry_count']):
try:
comm = open('article' + conf['comm_file'], 'w')
comm.write(text)
comm.close()
return None
except (KeyboardInterrupt, SystemExit):
raise
except:
time.sleep(conf['retry_delta'])