def del_vm_by_name(session, name):
vids = xen_helper.get_vid_by_name(session, name)
if len(vids) == 0:
log("no vm named '" + name + "' exists!")
else:
for vid in vids:
xen_helper.del_vm_by_id(vid)
def _get_nodes_in_stringIO(data, id="<unknown ID>"):
"""Parse a file or stringIO object in the tracejob format
and return the list of nodes. It is useful to pass data
and open filename in the invocation so this can be tested
with plain strings instead of files."""
h.log("Parsing...")
group_of_nodes = []
for line in data:
if "exec_vnode=" in line:
nodes = []
stuff = line.split()
for entry in stuff:
if entry.startswith("exec_vnode="): # exec_vnode=(r10i1n26:ncpus=36)+(r10i1n31:ncpus=36)
pbs_nodes = entry.split("=(")[1] # r10i1n26:ncpus=36)+(r10i1n31:ncpus=36)
pbs_node_list = pbs_nodes.split(")+(") # r10i1n26:ncpus=36 , r10i1n31:ncpus=36)
for n in pbs_node_list:
nodes.append(n.split(":")[0]) # r10i1n26 , r10i1n31
group_of_nodes.append(nodes)
tbr = []
for group in group_of_nodes:
if group not in tbr:
tbr.append(group)
if len(tbr) > 1:
print "WARNING: job", id, "ran", len(tbr), "times"
h.log("Parsing completed")
return tbr
def main_loop(self):
try:
self.sock.connect((HOST, PORT))
self.sock.send("PASS {}\r\n".format(PASS).encode("utf-8"))
self.sock.send("NICK {}\r\n".format(NICK).encode("utf-8"))
self.sock.send("JOIN {}\r\n".format(CHAN).encode("utf-8"))
self.connected = True
self.chat('hi all o/')
except Exception as ex:
self.connected = False
print('main_loop 1:', ex)
while self.connected:
try:
self.getUsers()
if self.executor() == self.EXECUTOR_STATE_DEAD:
helper.log('main_loop 2: EXECUTOR FAILED!')
break
except Exception as ex:
self.connected = False
print('main_loop 3:', ex)
sleep(0.1)
helper.log('PASSED AWAY')
def chat(self, msg, memoryLifeTime=30, chatDelay=0):
"""
:type msg: string
:type memoryLifeTime: int
:type chatDelay: int
"""
if self.chatMemory.isInMemory(msg):
helper.log('MEMORY BLOCK FOR: "' + msg + '"')
return False
if self.isSilent:
helper.log('stealth-mode: "' + msg + '"')
return False
self.chatMemory.add(msg, memoryLifeTime)
self.addMessageQueue(msg)
if chatDelay > 0:
time.sleep(chatDelay)
# for commands that rely on a timeout send one instantly for an outer-function sleep-loop
# TODO: hope that there is nothing else in the buffer ^^"
return self.sendMessageQueue()
def start(action):
if action == 'common':
start_spider()
elif action == 'hot':
start_hot()
helper.log('done', platform)
def cmd(self, str_send, wait=True, crop=True, binary=False):
str_recv = "" # response buffer
str_stat = "" # status buffer
token = c.DELIMITER + str(random.randrange(2**16)) # unique delimiter
footer = '@PJL ECHO ' + token + c.EOL + c.EOL if wait else ''
status = '@PJL INFO STATUS' + c.EOL if self.status and wait else ''
# send command to printer device
try:
cmd_send = c.UEL + str_send + c.EOL + status + footer + c.UEL
# write to logfile
log().write(self.logfile, str_send + os.linesep)
# sent to printer
self.send(cmd_send)
# for commands that expect a response
if wait:
# use random token as delimiter PJL responses
str_recv = self.recv('(@PJL ECHO\s+)?' + token + '.*$', wait,
True, binary)
if self.status:
# get status messages and remove them from received buffer
str_stat = item(
re.findall("@PJL INFO STATUS.*", str_recv, re.DOTALL))
str_recv = re.compile('\x0c?@PJL INFO STATUS.*',
re.DOTALL).sub('', str_recv)
if crop:
# crop very first PJL line which is echoed by most interpreters
str_recv = re.sub(r'^(\x00+)?@PJL.*' + c.EOL, '', str_recv)
return self.pjl_err(str_recv, str_stat)
# handle CTRL+C and exceptions
except (KeyboardInterrupt, Exception) as e:
if not self.fuzz or not str(e): self.reconnect(str(e))
return ""
def download_wallpaper(width=1920, height=1080):
"""Return the downloaded wallpaper file name"""
# available categories
categories = get_categories()
category = random.choice(categories)
# image url
base_url = "https://source.unsplash.com"
tags = get_tags()
collection = 1598183
#image_url = '{}/{}x{}?{},{}'.format(base_url,
# str(width), str(height), category, tags)
image_url = '{}/{}/{}/{}x{}'.format(base_url, "collection", str(collection), str(width), str(height))
#image_url = "https://source.unsplash.com/collection/1598187/1920x1080"
# landscapes
#image_url = "https://source.unsplash.com/collection/1598189/1920x1080"
# my first collection (everything)
#image_url = "https://source.unsplash.com/collection/1598183/1920x1080"
# download image
try:
wallpaper_name = 'wallpaper.jpg'
response = download_image(image_url, base_path("/" + wallpaper_name))
except:
# if previously download wallpaper not exist
if not os.path.isfile(base_path("/" + wallpaper_name)):
wallpaper_name = "default-wallpaper.jpg" # show the default wallpaper
log('Unable to download wallpaper')
return base_path("/" + wallpaper_name)
def clean_spec_chars(dir):
phonebookDir = get_main_dir()
dirname = dir['name']
files = dir['files']
enc = 'utf-8'
for file in files:
# # only use utf-8 converted files
# if('utf-8' not in file):
# continue
# clean only backup file
info = f'Cleaning {dirname} {file}'
log('INFO', info)
cleaned_data = []
with open(phonebookDir + dirname + file, 'r', encoding=enc) as data:
line = data.readline()
while line:
line = check_substring(line)
cleaned_data.append(line)
line = data.readline()
with open(phonebookDir + dirname + file + '_new', 'w', encoding=enc) as data:
for entry in cleaned_data:
data.write('%s\n' % entry)
def score(session):
if not session:
raise Exception("Invalid session for the request.")
data = getScorePageData(session)
if checkDone(data):
if confirm(
"It seem you have been finished the marking operation. Do you want to continue anyway?", nevigate=True
):
raise Exception("User has cancelled manually.")
term = currentTerm(data)
if not term:
raise Exception("Cannot found the term from the response data.")
t = teachers(data)
if not t:
raise Exception("Empty teacher list, nothing found.")
for teacher in t:
if teacher[settings.FIELD_TEAHCER_STATUS] == settings.RESULT_ALREADY_SCORE and settings.IGNORE_ALREADY_SCORED:
log(
Fore.YELLOW
+ "Ignore performing for teacher %s who has been marked." % teacher[settings.FIELD_TEACHER_NAME]
)
continue
log("Perform the marking operation for teacher %s." % teacher[settings.FIELD_TEACHER_NAME])
mark(teacher[settings.FIELD_TEACHER_ID], term, session)
def train(self):
args = self._args
self.model = model.GOAT(num_nodes=self.data.num_nodes,
emb_dim=args.dim)
self.model.to(device)
optimizer = torch.optim.Adam(self.model.parameters(),
lr=args.learning_rate)
total_batches_ = self.data.total_batches
bunch = total_batches_ / 100
intervals = np.arange(0, total_batches_, bunch)
helper.log(f'Total batches per epoch {total_batches_}')
for i in range(args.epochs):
for batch in self.data.train_batches:
self._infer(batch)
criterion = self.loss_fun(self.model)
optimizer.zero_grad()
criterion.loss.backward()
optimizer.step()
if args.dev_rate > 0:
self._validate(train_loss=criterion.loss.data, epoch=i)
else:
helper.log("Epoch {}/{} training loss = {:.5f}".format(
i + 1, args.epochs, criterion.loss.data))
def run(self):
pattern = re.compile(
r'<div\sclass="product\-card"\sid="\w+"\sdata\-prodid="\w+"\sdata\-productid="\w+"\sdata\-baseurl="[\/\w\-\?]*">\s+<a\sid="\w+"\shref="[\/\w\-\?&=]*"'
)
try:
html = helper.post(
self.url,
self.post_body, {
'origin': 'https://www.finishline.com',
'referer':
'https://www.finishline.com/store/men/shoes/_/N-1737dkj?mnid=men_shoes&Ns=sku.bestSeller%7C1&sort=sort%3Abest%20sellers%0A%20',
'user-agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36 OPR/52.0.2871.40',
'x-requested-with': 'XMLHttpRequest',
'accept': '*/*',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'zh-CN,zh;q=0.9',
'content-length': '0'
},
returnText=True)
str_arr = pattern.findall(html)
for s in str_arr:
self.q.put('https://www.finishline.com%s' %
s.split(' href="')[1].replace('"', ''))
except:
helper.log('[ERROR] => ' + self.url, 'finishline')
self.error_page_url_queue.put({
'url': self.url,
'count': self.count,
'post_body': self.post_body,
'gender': self.gender
})
def filter_col(table, col, func):
# Returns a list of indexes of col entries that is true on the function
hpr.log(TESTING, 'filter_col table keys: {0}'.format(table.keys()))
if not has_col(table, col):
raise ValueError('Column {0} not in table'.format(col))
entries = table[col]
return [i for i in range(len(entries)) if func(entries[i])]
def teachers(data):
"""
Retrieve the teacher information
"""
# pattern = re.compile('<option[^(]+? >')
pattern = re.compile(r'<option[^()"]+?\s+>.+?</option>')
# <option value=id>xxx</option> list
raw_teachers = pattern.findall(data)
if len(raw_teachers) == 0:
return False
t = []
for raw_t in raw_teachers:
# Split to multi groups
pattern = re.compile(r"<.+?>")
parts = pattern.findall(raw_t)
if len(parts) < 3:
continue
teacherId = re.search(r"[^(<option value=)]+", parts[0]).group()
teacherName = re.search(r"[^<>]+", parts[1]).group()
status = re.search(r"[^<>]+", parts[2]).group()
log(Fore.GREEN + "Found id %s for teacher %s with status %s." % (teacherId, teacherName, status))
t.append(
{
settings.FIELD_TEACHER_ID: teacherId,
settings.FIELD_TEACHER_NAME: teacherName,
settings.FIELD_TEAHCER_STATUS: status,
}
)
return t
def runWindows(nameString, vendorString):
accentuate("Starting Windows Algorithm")
setBackground()
log("Set computer background.")
numDeleted = uninstallUselessPrograms(nameString, vendorString)
log("Completed Windows algorithm.")
return {'programs-uninstalled': numDeleted}
def start():
crawl_counter = mongo.get_crawl_counter(platform)
# 创建一个队列用来保存进程获取到的数据
q = Queue()
# 有错误的页面链接
error_page_url_queue = Queue()
# 先获取cookie
_, tmpCookie = helper.get('https://www.kickz.com/us/men/shoes/c', myHeaders={
'User-Agent': 'Mozilla/5.0'
}, withCookie=True)
global cookie
cookie['JSESSIONID'] = tmpCookie.get('JSESSIONID', '')
total_page = 20
fetch_page(['https://www.kickz.com/us/men/shoes/c?selectedPage=%d' % page
for page in range(1, total_page + 1)], 1, q, error_page_url_queue, crawl_counter)
total_page = 17
fetch_page(['https://www.kickz.com/us/kids,women/shoes/shoe-sizes/38+,36-2:3,40+,37+,41+,39-1:3,35+,36,36+,39+,39,37,38,41-1:3,42,41,40,39:40,38-2:3,40-2:3,35:36,37:38,37-1:3,41:42/c?selectedPage=%d' % page
for page in range(1, total_page + 1)], 2, q, error_page_url_queue, crawl_counter)
# # 处理出错的链接
# while not error_page_url_queue.empty():
# error_page_url_list = []
# while not error_page_url_queue.empty():
# error_page_url_list.append(error_page_url_queue.get())
# error_page_men_url_list = [url_data.get('url') for url_data in error_page_url_list if url_data.get('gender') == 1]
# fetch_page(error_page_men_url_list, 1, q, error_page_url_queue, crawl_counter)
# error_page_women_url_list = [url_data.get('url') for url_data in error_page_url_list if url_data.get('gender') == 2]
# fetch_page(error_page_women_url_list, 2, q, error_page_url_queue, crawl_counter)
helper.log('done', platform)
def givenote():
data = None
try:
nic = request.form.get('nic').lower()
key = request.form.get('key').lower()
fname = request.form.get('fname').lower()
except:
return ret_status(1, route=str("/download/note from %s" % request.remote_addr))
log_key = helper.search_key(nic)
if log_key!= None:
if log_key.split('^')[1] == key:
try:
f = open('%s/users/%s/%s' % (work_dir, nic, fname), "r").read().splitlines() # Open for reading
content = ''
for s in f:
content += s + '\n'
data = {'content':content, 'fname':fname}
log('Sending ./%s/users/%s/%s to %s' % (work_dir, nic, fname, request.remote_addr))
code = 0
except:
code = 4
else:
code = 2
else:
code = 3
return ret_status(code, data=data, route=str("/download/note from %s" % request.remote_addr))
def start(action):
if action == 'common':
crawl_counter = mongo.get_crawl_counter('champssports')
# 创建一个队列用来保存进程获取到的数据
q = Queue()
# 有错误的页面链接
error_page_url_queue = Queue()
total_page = 16
# https://www.champssports.com/api/products/search?products=&query=%3Arelevance%3Agender%3A200000%3AproductType%3A200005¤tPage=1&pageSize=60×tamp=0
base_url = 'https://www.champssports.com/Mens/Shoes/_-_/N-24Zrj?cm_PAGE=%d&Rpp=180&crumbs=991&Nao=%d'
fetch_page([base_url % ((page - 1) * 180, (page - 1) * 180) for page in range(1, total_page + 1)], 1, q, error_page_url_queue, crawl_counter)
total_page = 7
base_url = 'https://www.champssports.com/Womens/Shoes/_-_/N-25Zrj?cm_PAGE=%d&Rpp=180&crumbs=991&Nap=%d'
fetch_page([base_url % ((page - 1) * 180, (page - 1) * 180) for page in range(1, total_page + 1)], 2, q, error_page_url_queue, crawl_counter)
# 处理出错的链接
while not error_page_url_queue.empty():
error_page_url_list = []
while not error_page_url_queue.empty():
error_page_url_list.append(error_page_url_queue.get())
error_page_men_url_list = [url_data.get('url') for url_data in error_page_url_list if url_data.get('gender') == 1]
fetch_page(error_page_men_url_list, 1, q, error_page_url_queue, crawl_counter)
error_page_women_url_list = [url_data.get('url') for url_data in error_page_url_list if url_data.get('gender') == 2]
fetch_page(error_page_women_url_list, 2, q, error_page_url_queue, crawl_counter)
helper.log('done', 'champssports')
def runCleaner():
accentuate("Starting Cleaner Algorithm")
numDesktopFiles = deleteAllDesktopShortcuts()
desktopFoldersDeleted = deleteAllFolders(desktopStr)
log("Deleted " + str(desktopFoldersDeleted) + " folders from the Desktop.")
documentsDeleted = deleteAllWrapper(documentsStr)
picturesDeleted = deleteAllWrapper(picturesStr)
videosDeleted = deleteAllWrapper(videosStr)
musicDeleted = deleteAllWrapper(musicStr)
downloadsDeleted = deleteAllWrapper(downloadsStr)
log("Completed the cleaner algorithm.")
return {
'desktop-files-deleted': numDesktopFiles,
'desktop-folders-deleted': desktopFoldersDeleted,
'document-files-deleted': documentsDeleted[0],
'document-folders-deleted': documentsDeleted[1],
'pictures-files-deleted': picturesDeleted[0],
'pictures-folders-deleted': picturesDeleted[1],
'videos-files-deleted': videosDeleted[0],
'videos-folders-deleted': videosDeleted[1],
'music-files-deleted': musicDeleted[0],
'music-folders-deleted': musicDeleted[1],
'downloads-files-deleted': downloadsDeleted[0],
'downloads-folders-deleted': downloadsDeleted[1],
}
def filter_col(table, col, func):
# Returns a list of indexes of col entries that is true on the function
hpr.log(TESTING, 'filter_col table keys: {0}'.format(table.keys()))
if not has_col(table, col):
raise ValueError('Column {0} not in table'.format(col))
entries = table[col]
return [i for i in range(len(entries)) if func(entries[i])]
def shape_traffic(self, params=None, dist_db=None):
if params != None:
pass
elif self.params != None:
params = self.params
else:
log("no traffic params is set!")
#log("shaping traffic for:"+str(self.link.name))
cmds = traffic_cmd_compile(params, dist_db=dist_db)
# determine linux system if name to apply control to
if_name = ""
if self.out:
new_ifb = ifb.create_new()
new_ifb.start()
#raw_input("after starting ifb")
self.link_if.redirect_to(new_ifb)
if_name = new_ifb.name
#raw_input("after redirecting to new ifb: "+if_name)
else:
if_name = self.link_if.name
#raw_input("shaping in traffic with vif")
for i in range(0, len(cmds)):
cmds[i] = (cmds[i].format(if_name))
#log(cmds)
#raw_input("in if_link.shape_traffic(): ")
info_exe(cmds)
def bounty_processing(directory, database):
d = {"PostId": [], "BountyAmount": []}
for event, elem in ET.iterparse(os.path.join(directory, "Votes.xml")):
if event == "end":
try:
if "BountyAmount" in elem.attrib:
postid = int(elem.attrib["PostId"])
bounty = int(elem.attrib["BountyAmount"])
d["PostId"].append(postid)
d["BountyAmount"].append(bounty)
elem.clear()
except Exception as e:
pass
DB = sqlite3.connect(database)
answers_meta = pd.read_sql(
'select AnswerId, QuestionId from "AnswersMeta"', DB)
DB.close()
question_bounty = {"QuestionId": [], "Bounty": []}
for postid, bounty in zip(d["PostId"], d["BountyAmount"]):
if answers_meta[answers_meta["QuestionId"] == postid].index.tolist():
question_bounty["QuestionId"].append(postid)
question_bounty["Bounty"].append(bounty)
df = pd.DataFrame(question_bounty)
write_table(database, "Question_Bounty", df)
statistics_file = os.path.join(Path(database).parent, "statistics.log")
log(statistics_file, "# questions having bounty: " + str(len(df)))
def _processes_train_dev_data(self):
"""
Builds training and validation batches
:return:
"""
args = self._args
self.dev_batches = []
train_edges = self._data.edges
if self._data.use_dev:
dev_size = int(len(self._data.train_sources) * args.dev_rate)
helper.log(f'Number of dev points: {dev_size}')
dev_edges = self._data.edges[:dev_size]
train_edges = self._data.edges[dev_size:]
self.dev_batches = _generate_input_batches(
neighborhood_matrix=self._data.neighborhood_matrix,
mask_matrix=self._data.mask_matrix,
edges=dev_edges)
self.train_batches = _generate_input_batches(
neighborhood_matrix=self._data.neighborhood_matrix,
mask_matrix=self._data.mask_matrix,
edges=train_edges)
self.total_batches = len(self.train_batches)
helper.log(f'Number of training points: {len(train_edges)}')
async def shell(self, ctx, *, cmd: str):
"""A system shell, does not support command persistance."""
p = await asyncio.create_subprocess_shell(
cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
msg = await ctx.send("Executing...")
try:
stdout, stderr = (None, None)
tbefore = time.perf_counter()
stdout, stderr = await asyncio.wait_for(p.communicate(), 360)
tafter = time.perf_counter()
except asyncio.TimeoutError:
log(content=f'Command "shell" timed out.')
await msg.edit(content="Process timed out.")
except asyncio.CancelledError:
log(content="Process cancelled.")
await msg.edit(content="Process cancelled.")
await p.terminate()
if p.stderr == None:
await msg.edit(
content=
f"```py\nExit code: {p.returncode}\n{decode(stdout)}\nTook {round((tafter-tbefore)*1000, 2)}ms```"
)
else:
await msg.edit(
content=
f"```py\nExit code: {p.returncode}\nStdout:\n{decode(stdout)}\nStderr:\n{decode(stderr)}\nTook {round((tafter-tbefore)*1000, 2)}ms```"
)
def start_all(self):
# start all devices
[self.node_list[devid].start(self.session) for devid in self.dev_set]
# start all routers
[self.node_list[rid].start() for rid in self.switch_set]
# start all prouters
[
self.node_list[prid].start(self.node_list)
for prid in self.router_set
]
# waiting for link control to apply after all machine network functions are ready
count = len(self.dev_set)
tap_check = [False] * count
log("looping to wait for network-ready on all vms before setting up traffic rules"
)
while count != 0:
i = 0
for devid in self.dev_set:
if not tap_check[i]:
if self.node_list[devid].check_tap():
#log("get the "+str(i)+"th tap ready for device "+self.node_list[devid].name)
tap_check[i] = True
count -= 1
i += 1
log("start applying traffic shaping rules...")
# enable link control
[
link.shape_traffic(dist_db=self.dist_db)
for link in self.controlled_link_set
]
def find_low():
stock_file = open('stocks.txt', 'r')
stock_lines = stock_file.readlines()
black_symbol_map = helper.get_black_symbol_map()
work_index = helper.get_runtime_config("work_index")
if work_index:
work_index = int(work_index)
stock_lines = stock_lines[work_index:(len(stock_lines) - 1)]
i = 0
for line in stock_lines:
start_index = line.find(":")
helper.save_runtime_config("work_index", i)
i += 1
if start_index < 1:
continue
name = line[0:start_index]
symbol = line[start_index + 1:(len(line) - 1)]
if black_symbol_map.get(symbol):
continue
data_source = XueQiu(symbol)
kline = data_source.get_quarter_kline()
if kline is None:
# 如果不能拿到K线,说明这个股票有问题,需要拉黑
helper.black_symbol(symbol)
continue
if kline.is_stop():
# 如果停牌,则忽略
helper.log("stock", name + " 已停牌")
continue
print name + ":" + symbol + " LP:" + str(kline.is_latest_low_point())
time.sleep(1)
def teachers(data):
'''
Retrieve the teacher information
'''
# pattern = re.compile('<option[^(]+? >')
pattern = re.compile(r'<option[^()"]+?\s+>.+?</option>')
# <option value=id>xxx</option> list
raw_teachers = pattern.findall(data)
if len(raw_teachers) == 0:
return False
t = []
for raw_t in raw_teachers:
# Split to multi groups
pattern = re.compile(r'<.+?>')
parts = pattern.findall(raw_t)
if (len(parts) < 3):
continue
teacherId = re.search(r'[^(<option value=)]+', parts[0]).group()
teacherName = re.search(r'[^<>]+', parts[1]).group()
status = re.search(r'[^<>]+', parts[2]).group()
log(Fore.GREEN + 'Found id %s for teacher %s with status %s.' %
(teacherId, teacherName, status))
t.append({
settings.FIELD_TEACHER_ID: teacherId,
settings.FIELD_TEACHER_NAME: teacherName,
settings.FIELD_TEAHCER_STATUS: status
})
return t
def score(session):
if not session:
raise Exception('Invalid session for the request.')
data = getScorePageData(session)
if checkDone(data):
if confirm(
'It seem you have been finished the marking operation. Do you want to continue anyway?',
nevigate=True):
raise Exception('User has cancelled manually.')
term = currentTerm(data)
if not term:
raise Exception('Cannot found the term from the response data.')
t = teachers(data)
if not t:
raise Exception('Empty teacher list, nothing found.')
for teacher in t:
if teacher[
settings.
FIELD_TEAHCER_STATUS] == settings.RESULT_ALREADY_SCORE and settings.IGNORE_ALREADY_SCORED:
log(Fore.YELLOW +
'Ignore performing for teacher %s who has been marked.' %
teacher[settings.FIELD_TEACHER_NAME])
continue
log('Perform the marking operation for teacher %s.' %
teacher[settings.FIELD_TEACHER_NAME])
mark(teacher[settings.FIELD_TEACHER_ID], term, session)
def get_new_vif_id(self):
# find out the empty slot in interface list as the first available vif slot
for i in range(0, len(self.if_lst)):
if self.if_lst[i] == None:
return i
log("all vif slots are occupied!", level=logging.CRITICAL)
return None
def stop_all_threads():
threads = [pct_thread, tmt_thread, wqt_thread, dlt_thread, smt_thread]
for thread in threads:
if thread.is_alive():
log(thread.name + ' is still alive')
thread.exit()
while thread.is_alive():
time.sleep(0.1)
def progress(self, progress, done):
if done:
helper.log(":")
else:
progress = int(progress)
# only shows progress if we've moved a significant enough amount
if (progress % 5 == 0):
helper.log(".", newline=False)
def stop_all_threads():
threads = [pct_thread, tmt_thread, wqt_thread, dlt_thread, smt_thread]
for thread in threads:
if thread.is_alive():
log(thread.name + ' is still alive')
thread.exit()
while thread.is_alive():
time.sleep(0.1)
def handle_500(e):
original = getattr(e, 'original_exception', None)
if original is None:
# direct 500 error, such as abort(500)
return 'direct 500 error', 500
# unhandled error
log(logging.ERROR, traceback.format_exc(original))
return '', 500
def delete(self, idx):
""" Deletes the file provided as index of the internal arrays """
fname = self.fnames[idx]
log('Deleting %s' % str(os.path.abspath(fname)))
os.remove(fname)
if os.path.isfile(fname):
log('Unable to remove file %s' % str(os.path.abspath(fname)))
self.__init__()
def deleteHistory():
filesDeleted = deleteAllFiles(chromeDataStr)
log("Deleted " + str(filesDeleted) + " Chrome files.")
numFolderDeleted = execute(
r'FOR /D %p IN ("' + chromeDataStr +
r'\*.*") DO (if not "%~np"=="Default" rmdir "%p" /s /q)', r'rmdir') - 1
log("Removed " + str(numFolderDeleted) + " Chrome storage folders.")
return [filesDeleted, numFolderDeleted]
def delete(self, idx):
""" Deletes the file provided as index of the internal arrays """
fname = self.fnames[idx]
log('Deleting %s' % str(os.path.abspath(fname)))
os.remove(fname)
if os.path.isfile(fname):
log('Unable to remove file %s' % str(os.path.abspath(fname)))
self.__init__()
def sendUpdate(data, url):
request = requests.patch(url, data, headers=headers)
log('API - ' + str(request))
# Testing
# updateTrigger(1)
# updateIP(1, '192.126.234.32')
def progress(self, progress, done):
if done:
helper.log(":")
else:
progress = int(progress)
# only shows progress if we've moved a significant enough amount
if (progress % 5 == 0):
helper.log(".", newline=False)
def remove_all_from_cloud_view(self):
""" remove all items from 'Cloud' view """
# switch to 'Cloud'
helper.log("switch to 'Cloud'")
helper.retry(self.switch_to_cloud_view)
time.sleep(2)
# remove all from current view
return self.remove_all()
def remove_all_from_mymac_view(self):
""" remove all items from ''My Mac' view """
# switch to 'My Mac'
helper.log("switch to 'My Mac'")
helper.retry(self.switch_to_mymac_view)
time.sleep(2)
# remove all from current view
return self.remove_all()
def control(device_name, gpio, state):
if state == 0:
if not config.SIMULATION:
set_gpio(gpio, 0)
else:
if not config.SIMULATION:
set_gpio(gpio, 1)
log('%s %d' % (device_name, state))
cook.status[device_name] = state
def get_nodes_in_file(filename):
h.log("\n---------------------------------------\nProcessing file " + str(filename))
try:
with open(filename) as f:
nodes = parse(f)
except IOError as ioe:
print ioe
sys.exit(1)
h.log("Ended processing file " + str(filename) + ", " + str(len(nodes)) + " nodes found.")
return nodes
def sign_out(self):
""" sign out from RT account setting menu """
helper.log("Click account setting button on top bar")
if self.region.exists(account_setting_button_top_bar, default_wait_time):
self.region.click(account_setting_button_top_bar)
helper.log("Click sign out button")
self.region.click(sign_out_button_in_account_setting_button_top_bar)
return True
return False, "Fail to find 'Account setting button on top bar of RealTimes"
def test_webdriver(self):
driver = self.driver
helper.setLogFile(LOG_FILE)
helper.log("browserStarted||"+str(TREATMENTID), ID)
run = 0
while (run < RUNS):
helper.applyTreatment(driver, TREATMENTS[TREATMENTID], ID, TREATMENTID)
helper.wait_for_others(AGENTS, ID, ROUND)
time.sleep(20)
helper.collectMeasurement(driver, MEASUREMENT, ID, TREATMENTID)
run = run+1
def manage():
global brew_recipe
if request.method == 'POST':
log('Manage: ' + str(request.form))
eval_manage_form(request.form)
brew_recipe = recipes.get_selected_recipe()
selected = recipes.get_selected_fname()
return render_template('manage.html',
heading='Manage',
fnames=recipes.fnames,
selected=selected)
def get_nodes_in_job(jobid):
"""Wrapper facade around the tracejob invocation logic and
output parser. Simply return the list of nodes
where a given jobID ran."""
h.log("\n---------------------------------------\nProcessing job " + str(jobid))
try:
#nodes = _get_nodes_in_stringIO(open(_invoke_tracejob(jobid)), id=jobid)
nodes = _get_nodes_in_stringIO(open(get_job_info(jobid)), id=jobid)
except Exception as e:
print e, "in job ID", jobid
nodes = []
h.log("Ended processing job " + str(jobid) + ", " + str(len(nodes)) + " nodes found.")
return nodes
def _invoke_tracejob(jobid):
"""Invoke tracejob for a specific jobid. TBD if storing the
output in a temporary file will be useful, but doing it
for now"""
name = "tracejob." + str(jobid) + ".txt"
fullname = h.TMPDIR + name
ALL_FILES = os.listdir(h.TMPDIR)
if not name in ALL_FILES:
command = "tracejob -n 50 " + str(jobid) + " > " + fullname
h.log("Invoking: " + command)
subprocess.call(command, shell=True)
else:
h.log("No need to invoke tracejob. Using cached " + fullname)
return fullname
def save(self, recipe):
""" Adds timestamps and saves the recipe provided as parameter """
if 'created' not in recipe or recipe['created'] == '':
recipe['created'] = str_timestamp_now()
recipe['last_saved'] = str_timestamp_now()
dir_entry_path = os.path.join(PATH, recipe['name'] + EXT)
log('Saving %s' % str(os.path.abspath(dir_entry_path)))
if isWritable(PATH):
with open(dir_entry_path, 'wb') as my_file:
pickle.dump(recipe, my_file)
else:
log('Error while saving. Cannot write to %s' %
str(os.path.abspath(dir_entry_path)))
self.__init__()
def share_media_in_library_view(self, name_list, share_to_account=account_username2):
"""
Share a media item or multiple media items to another account.
Currently we use default account_username to share
"""
if self.region.exists(photos_and_videos_button_selected):
self.region.click(photos_and_videos_button_selected)
else:
return False, "Fail to find 'Photos & Videos' button"
# Select an album to share
helper.log("The media item(s) will be shared")
helper.multiple_select_by_cliclick(name_list)
return self.share_media(share_to_account)
def share_album(self, album_item, album_name='', share_to_account=account_username2):
""" Share an album to another account, currently we use default account_username to share """
helper.log("Switch to album")
helper.retry(self.switch_to_albums_view)
# Select an album to share
helper.log("The created album will be shared")
if self.region.exists(album_item, default_wait_long_time):
self.region.click(album_item)
elif helper.check_exist_by_cliclick(album_name):
helper.click_item_by_cliclick(album_name)
else:
return False, "The album item doesn't exist"
helper.assert_step(self.share_via_bottom_bar())
time.sleep(20)
return self.share_dialog(share_to_account)
def currentTerm(data):
"""
Extract the current term from the score page, such as 2015/2016(2), it's necessary for the score operation
"""
pattern = re.compile("<option.+?selected>")
raw_term = pattern.search(data)
if raw_term == None:
return False
term = re.search("[0-9]{4}/[0-9]{4}\([0-9]+\)", raw_term.group(0))
if term == None:
return False
term = term.group(0)
log(Fore.GREEN + "Retrieve current term: %s" % term)
return term
def run():
global pct_thread
global default_str_time
# if not pct_thread or not pct_thread.is_alive():
# raise RuntimeError('ProcControlThread is not running')
if request.method == 'POST':
log('Run: ' + str(request.form))
if 'btn_start' in request.form:
pct_thread.start_cooking(brew_recipe)
dlt_thread.start_logging()
elif 'btn_start_at' in request.form:
# Calculate datetime object when cooking should start
str_time = request.form['start_time']
default_str_time = str_time
start_at_hour = int(str_time.split(':')[0])
start_at_min = int(str_time.split(':')[1])
now = datetime.datetime.now()
start_at = datetime.datetime.now().replace(hour=start_at_hour,
minute=start_at_min,
second=0,
microsecond=0)
# Add a day when start_at is in the past
if now > start_at:
start_at += datetime.timedelta(days=1)
pct_thread.start_cooking(brew_recipe, start_at)
dlt_thread.start_logging()
elif 'btn_stop' in request.form:
pct_thread.stop_cooking()
dlt_thread.stop_logging()
elif 'btn_t1_up' in request.form:
tpc.inc_offset(0.2)
elif 'btn_t1_down' in request.form:
tpc.inc_offset(-0.2)
else:
pass # unknown
# When cooking is started always use the recipe that is currently running
if cook.status['pct_state'] == 'Waiting' or \
cook.status['pct_state'] == 'Running':
cur_recipe = cook.status['recipe']
else:
cur_recipe = brew_recipe
return render_template('run.html', heading='Run', start_at=default_str_time,
state=cook.status, data=cur_recipe)
def upload(self, flickr, img_fn, meta):
fn=meta["FileName"]
caption=meta["Caption"]
# changes tags to lowercase
tags=meta["Keywords"].lower()
try:
kw = dict(title=fn,
tags=tags,
is_public=self.__is_public,
is_friend=self.__is_friend,
is_family=self.__is_family,
hidden=self.__hidden,
format="etree")
helper.log("%s " % img_fn, newline=False)
return flickr.upload(filename=str(img_fn), callback=self.progress, **kw).find("photoid").text
except Exception, e:
traceback.print_exc(sys.stderr)
return None
def read_sensor(path):
''' reads the value from device in the given path
return a float, the sensor value when successful, -99.0 otherwise
'''
try:
fd = open(path, "r")
line = fd.readline()
if re.match(r"([0-9a-fd]{2} ){9}: crc=[0-9a-fd]{2} YES", line):
line = fd.readline()
m = re.match(r"([0-9a-fd]{2} ){9}t=([+-]?[0-9]+)", line)
value = float(m.group(2)) / 1000.0
else:
log('No RegEx match with %s' % line)
value = INV_TEMP
fd.close()
except (IOError), e:
log('Error reading %s:%s' % (path, e))
# As an error value use a higher than normal temperature to avoid that
# the heater is switched on. E.g. with negative values the heater would
# start to raise the temperature even more.
value = INV_TEMP
def proc_consult_file(consult_file, result_file):
tbs, consults = cns.get_consults(consult_file)
result_f = open(result_file, 'a')
for i in range(len(consults)):
c = consults[i]
hpr.log(TESTING, 'At consult: {0}'.format(c))
to_file = '------- CONSULTA {0} -------\n'.format(i + 1)
try:
to_file += man.printable(cns.raw_query(c, tbs))
except ValueError as ve:
print('Error: {0}'.format(ve))
to_file += 'FALLIDA'
except hpr.InvalidQueryError as iqe:
print('Error: {0}'.format(iqe))
to_file += 'FALLIDA'
except hpr.TerminateQueryError as tqe:
print('Query terminated: {0}'.format(tqe))
# Append an empty line
to_file += '\n'
except Exception as e:
print('Error: {0}'.format(e))
to_file += 'FALLIDA'
finally:
print('Completed consult {0}.'.format(i + 1))
print(to_file, end='\n', file=result_f)
hpr.log(TESTING, '\nConsult {0}: {1}'.format(i + 1, c))
hpr.log(TESTING, to_file)
result_f.close()
return tbs
def main():
init(autoreset = True)
print Fore.GREEN + '============================================'
print Fore.GREEN + ' Auto-Score by Soxfmr '
print Fore.GREEN + ' ver 0.2 20151128 '
print Fore.GREEN + '============================================'
try:
username = require('Sutdent Id: ')
password = require('Password: '******'Do you want to ignore the marked record')
settings.IGNORE_ALREADY_SCORED = ignore
if username == '' or password == '':
raise Exception('Invalid input value.')
# Retrieve the user session
log('Preparing the user session...')
session = login(username, password)
# Begin
log(Fore.GREEN + 'Session established. Getting start to marking...')
score(session)
log(Fore.GREEN + 'All done! Now you should login to the education system and confirm all of record!', important = True)
except Exception as e:
print Fore.RED + e.message
def remove_all(self):
retry = 5
while not self.region.exists(no_photos_or_videos, default_wait_time):
# select all
self.region.click(select_current_view)
type("a", Key.CMD)
# click delete button
type(Key.DELETE)
# if current view contains cloud&local content, will select 'delete
# from this mac' radio button
if self.region.exists(delete_from_this_mac_radio_button, default_wait_time):
self.region.click(delete_from_this_mac_radio_button)
time.sleep(2)
type(Key.ENTER)
retry -= 1
if retry < 0:
break
else:
helper.log("No photos or videos, return")
return True
def monitor(self, bot, l_lock, t_lock):
self.update()
while(1):
while not self.empty():
paste = self.get()
self.ref_id = paste.id
with l_lock:
helper.log('[*] Checking ' + paste.url)
paste.text = helper.download(paste.url)
with l_lock:
tweet = helper.build_tweet(paste)
if tweet:
print tweet
with t_lock:
helper.record(tweet)
bot.PostUpdate(tweet)
self.update()
# If no new results... sleep for 5 sec
while self.empty():
with l_lock:
helper.log('[*] No results... sleeping')
sleep(SLEEP_SLEXY)
self.update()
def edit():
global brew_recipe
last_action = 'Empty'
if request.method == 'POST':
log('Edit: ' + str(request.form))
if 'btn_add_row' in request.form:
brew_recipe['list'].append((0.0,0))
last_action = 'Zeile hinzu'
elif 'btn_del_row' in request.form:
brew_recipe['list'].pop(-1)
last_action = u'Zeile löschen'
elif 'btn_save' in request.form:
eval_edit_form(request.form, brew_recipe)
recipes.save(brew_recipe)
last_action = 'Speichern'
elif 'btn_reset' in request.form:
brew_recipe = recipes.get_default()
last_action = u'Zurücksetzen'
else:
pass
return render_template('edit.html',
heading='Edit',
data=brew_recipe,
last_action=last_action)
def commandAddPodcast(args):
"""add a new feed-url to database
"""
try:
cast = feedparser.parse(args.url)
except:
log("Couldn't parse. (%s)" % args.url)
else:
with DB() as dbHandler:
dbHandler.sql(
"INSERT INTO casts (\
title, url, last_updated, status, short_title\
) VALUES (?,?,?,?,?)", (
cast.feed.title,
args.url,
now()[1],
STATUS_UPDATE_CAST,
args.short_title
)
)
feedId = dbHandler.getLastId()
cast = Cast(feedId)
cast.update()
cast.get_image()
def monitor(self, bot, l_lock, t_lock):
self.update()
while(1):
while not self.empty():
paste = self.get()
self.ref_id = paste.id
with l_lock:
helper.log('[*] Checking ' + paste.url)
# goober pastie - Not actually showing *raw* text.. Still need to parse it out
paste.text = BeautifulSoup(helper.download(paste.url)).pre.text
with l_lock:
tweet = helper.build_tweet(paste)
if tweet:
print tweet
with t_lock:
helper.record(tweet)
bot.PostUpdate(tweet)
self.update()
# If no new results... sleep for 5 sec
while self.empty():
with l_lock:
helper.log('[*] No results... sleeping')
sleep(SLEEP_PASTIE)
self.update()
