def prepare_data(self):
"""Data prep. To be run once."""
# Download & Extract training & validation dataset
if os.path.isdir("BSD300"):
logger.debug("BDS300 already exists.")
else:
# Download
fname = wget.download(C.BSDS_URL)
# Extract
with tarfile.open(fname) as tar:
tar.extractall()
# Cleanup
os.remove("BSDS300-images.tgz")
# Download & Extract testing dataset
if os.path.isdir("cifar-10"):
logger.debug("cifar-10 already exists.")
self.test_dataset = CIFAR10(root_dir="cifar-10",
train=False,
image_size=C.IMG_SIZE,
sigma=C.SIGMA,
download=False)
else:
# Download
# noinspection PyAttributeOutsideInit
self.test_dataset = CIFAR10(root_dir="cifar-10",
train=False,
image_size=C.IMG_SIZE,
sigma=C.SIGMA,
download=True)
def login(account: dict):
client = amino.Client()
email = account.get("email")
password = account.get("password")
while True:
try:
client.login(email, password)
return client
except amino.exceptions.ActionNotAllowed:
client.device_id = client.headers.device_id = random.choice(
open(
os.path.join(os.getcwd(), "src", "devices", "devices.txt"),
"r").readlines()).replace("\n", "")
except amino.exceptions.FailedLogin:
service_log(email, "Failed login")
return False
except amino.exceptions.InvalidAccountOrPassword:
service_log(email, "Invalid account or password")
return False
except amino.exceptions.InvalidPassword:
service_log(email, "Invalid Password")
return False
except amino.exceptions.InvalidEmail:
service_log(email, "Invalid Email")
return False
except amino.exceptions.AccountDoesntExist:
service_log(email, "Account does not exist")
return False
except amino.exceptions.VerificationRequired as verify:
service_log(email, str(verify))
return False
except Exception as e:
service_log(email, str(e))
logger.debug(traceback.format_exc())
return False
def exportFasta(data, filename, number=0):
"""
create a fasta file
"""
with open(filename, 'w') as fileFasta:
if number < 0:
#gene random draw
listGenes = []
number = abs(number)
count = number
while count != 0:
#random draw in data and make sur of gene have a sequence
idGene = random.choice(list(data))
if data[idGene].sequence != "" and idGene not in listGenes:
listGenes.append(idGene)
count -= 1
else:
listGenes = data.keys()
count = 1
for gene in listGenes:
if data[gene].sequence != "":
if number == 0:
line = fasta(data[gene].name, data[gene].sequence)
fileFasta.write(line + '\n')
count += 1
if number > 0 and count <= number:
line = fasta(data[gene].name, data[gene].sequence)
fileFasta.write(line + '\n')
count += 1
log.debug("gene " + data[gene].name + " exported")
log.info(str(count - 1) + " gene(s) exported")
def process_udp_bytes(self, byte_list):
"""
see accessing-the-rako-bridge.pdf for decoding
:param byte_list: List(Int)
:return (topic: str, mqtt_payload: dict)
"""
logger.debug(f'received byte_list: {byte_list}')
if byte_list[0] != 83:
logger.debug('not a status update - ignore')
return
if byte_list[1] == 7:
if byte_list[5] != 49:
logger.debug('not a SET_SCENE command - ignore')
return
topic, payload = self.process_set_scene_bytes(byte_list)
elif byte_list[1] == 5:
if not (3 <= byte_list[5] <= 6) and byte_list[5] != 0:
logger.debug('not a SET_SCENE number command - ignore')
return
topic, payload = self.process_set_scene_number_bytes(byte_list)
else:
logger.debug('unhandled bytestring length - ignore')
return
return topic, payload
def beep(time=0.2, BUZZER_PIN=32):
logger.debug(f'buzzer pin {BUZZER_PIN}, time {time}')
safe_mode()
GPIO.setup(BUZZER_PIN, GPIO.OUT)
GPIO.output(BUZZER_PIN, GPIO.LOW)
sleep(time)
GPIO.cleanup(BUZZER_PIN)
def _check_auth_status(self):
res = requests.get(f"{API_URL}/iserver/auth/status", verify=False, timeout=REQUESTS_TIMEOUT_SEC)
status_content = json.loads(res.content)
L.debug(f"AUTH status: {status_content}")
return status_content["authenticated"] and status_content["connected"] and not status_content["competing"]
def __get_next_node_id(cls, data, state, node_data):
next_node_id = node_data.get('NextNodeId', '') # Fallback node id
for button in node_data.get('Buttons', []):
try:
root_key = re.split(r'\.|\[',
button.get("ConditionMatchKey"))[0]
if data.get(root_key) is None:
data[root_key] = None
logger.debug("rootKey " + root_key)
path = button.get("ConditionMatchKey")
obj = {root_key: data[root_key]}
variable_value = Util.deep_find(obj, path)
match_operator = button.get("ConditionOperator")
match_value = AnaHelper.verb_replacer(text=button.get(
"ConditionMatchValue", ""),
state=state)
condition_matched = AnaHelper.is_condition_match(
variable_value, match_operator, match_value)
if condition_matched:
next_node_id = button["NextNodeId"]
break
except:
logger.error("error in btn " + str(button))
pass
return next_node_id
def merge_dicts(*args):
result = {}
for dictionary in args:
logger.debug("Merging dictionaries")
logger.debug(f"{dictionary} {dictionary.__class__}")
result.update(dictionary)
return result
def run():
br = RakoBridge()
logger.debug(f'Running the rakomqtt {app_config.mode}')
if app_config.mode == "state_watcher":
run_state_watcher(br)
if app_config.mode == "commander":
run_commander(br)
def unlock(self):
if self.locked:
logger.debug(f"unlocking servo")
beep(0.1)
sleep(0.1)
beep(0.1)
self.move_servo(self.SERVO_MAX_VALUE)
self.locked = False
def safe_mode():
actual_mode = GPIO.getmode()
logger.debug(f"actual_mode {actual_mode}")
if actual_mode == -1 or actual_mode is None:
logger.debug(f"safe_mode setting BOARD")
GPIO.setmode(GPIO.BOARD)
return actual_mode
def benchmark_over(self, test_files) -> List[TestResult]:
results: List[TestResult] = list()
for test_file in test_files:
logger.debug(f"testing on {test_file}")
results.append(
test_program(self.executable_path, self.type, test_file,
self.count))
return results
def lock(self):
if not self.locked:
logger.debug(f"locking servo")
beep(0.1)
sleep(0.1)
beep(0.1)
self.move_servo(self.SERVO_MIN_VALUE)
self.locked = True
def postloop(self):
# 销毁所有session
ids = [str(i) for i in self.manager.session_map]
if len(ids):
self.manager.exec(['sessions', '-f', '-k', *ids])
logger.debug(f'All sessions `{str(ids)}` have been killed!')
self.manager._hook_destroy()
logger.debug("All the exploit destroy hooks on manager session are executed.")
super().postloop()
def verifFile(filename):
if os.path.exists(filename) and not args.force:
log.warning(filename + " already exist, you can overwrite with -f option")
return False
elif os.path.exists(filename) and args.force:
log.debug(filename + " will be overwrite")
return True
else:
return True
def create_connection(conn_string):
logger.debug("Starting ElasticSearch client")
try:
es = Elasticsearch([conn_string], sniff_on_start=True)
except:
raise ConnectionError(f"Couldn't connect to Elastic Search instance at: {conn_string} \
Check if you've started it or if it listens on the port listed above.")
logger.debug("Elasticsearch connected")
return es
def on_message(client, userdata, msg: mqtt.MQTTMessage):
m = re.match('^rako/room/([0-9]+)/set$', msg.topic)
if not m:
logger.debug(f"Topic unrecognised ${msg.topic}")
return
room__id = int(m.group(1))
payload_str = str(msg.payload.decode("utf-8"))
payload = serder.deserialise(payload_schema, payload_str)
rako_bridge.post_scene(room__id, payload['brightness'])
def move_servo(self, value):
assert type(value) is int
parsed_value = max(value, self.SERVO_MIN_VALUE)
parsed_value = min(parsed_value, self.SERVO_MAX_VALUE)
logger.debug(f'moving servo {self.SERVO_PIN}, value {parsed_value}')
pwm_servo = self.setup_servo_pin()
pwm_servo.ChangeDutyCycle(parsed_value)
sleep(0.3)
GPIO.cleanup(self.SERVO_PIN)
def cmdhandler(self, args: Cmdline)-> int:
if args.cmd is None:
return Command.STOP
cmd = args.cmd
func = self.session.command_map.get(cmd)
if func is None:
logger.error(f'No command named `{cmd}`.')
return Command.STOP
ret = func.run(args)
logger.debug(f'A command line `{args.cmdline}` is executed, returns `{ret}`')
return ret
def ping(self):
"""
Checks if Elastic is healthy
Returns:
True if healthy, False otherwise
"""
ping = self.es.ping()
if ping:
logger.debug(f'Ping successful')
return ping
def handle_events(self, events):
responses = []
for event in events:
logger.debug(f"Event received {event}")
event_type = EventType.get_name(event.get("type"))
handler_method = getattr(self, "handle_%s" % event_type.lower(), None)
if handler_method is None:
logger.error(f"Unknown event encountered in message {event}")
else:
response = handler_method(event) # for synchronous events, return the response
responses.extend(response)
return responses
def post_scene(self, room_id, brightness):
scene = self._rako_scene(brightness)
payload = {
'room': room_id,
'ch': 0,
'com': self.scene_to_scene_command[scene]
}
try:
logger.debug('payload {}'.format(payload))
requests.post(self._url, params=payload, timeout=self._timeout)
except Exception as ex:
logger.error("Can't turn on %s. Is resource/endpoint offline?",
self._url)
def run(keyword):
"""
启动搜索
"""
logger.debug("searching...")
loop = asyncio.get_event_loop()
tasks = [
search(item['name'], item['url'], keyword, item['selector'],
item['description']) for item in MOVIE_SITES
]
results = []
if tasks:
results = loop.run_until_complete(asyncio.gather(*tasks))
logger.debug("completed...")
return results
def __make_api_call(self, node_data):
logger.debug(f"State passed to replace api_url is {self.state}")
api_url = node_data.get("ApiUrl", "")
api_method = node_data.get("ApiMethod")
api_url = AnaHelper.verb_replacer(text=api_url, state=self.state)
logger.debug(f"URL after verb replacing is {api_url}")
api_headers = {}
headers = node_data.get("Headers", "").split("\n")
for header_line in headers:
header_key_values = header_line.split(":")[:2]
if len(header_key_values) == 2:
api_headers[header_key_values[0]] = header_key_values[1]
logger.debug(f"api headers: {api_headers}")
api_body = node_data.get("RequestBody", None)
if api_body:
api_body = AnaHelper.verb_replacer(text=api_body, state=self.state)
try:
api_body = json.loads(api_body)
except ValueError:
logger.error(f"Could not parse api body {api_body}")
return {}
logger.debug(f"api_body: {api_body}")
response = requests.request(method=api_method,
url=api_url,
headers=api_headers,
json=api_body)
logger.debug(f"api response: {response}")
if response.status_code in [200, 201]:
try:
api_response = response.json()
except ValueError:
api_response = response.text
else:
api_response = None
logger.error(
f"ApiCall did not return status code 200 {node_data['Id']}")
return api_response
def SendToServer(self, action: Dict) -> bool:
if self.socket is None:
logger.error(f"connection closed abrubtly on port: {self.port}")
return False
message: str = ""
try:
message = ServerState.Serialize(action)
self.socket.sendto(message.encode(), (self.host, self.port))
except Exception as e:
logger.error(f"error sending to server on port {self.port}\n{e}")
if self.verbose:
logger.debug(f"{message}")
return False
return True
def tblastn(file, blast):
"""
parse a xml file result of a tblastn
"""
count = 0 #counter of result in blast xml
#verification if file is a xml
f = open(file, 'r')
header = f.readline()
if '<?xml version="1.0"?>' not in header:
log.critical("Make sure of your file is an xml")
return False
f.close()
#initialisation for parsing file
tree = ET.parse(file)
root = tree.getroot()
for iteration in root.findall('./BlastOutput_iterations/Iteration'):
#a iteration is a query sequence
numberQuery = int(iteration[0].text)
name = iteration[2].text
lenght = iteration[3].text
blast[numberQuery] = Query(numberQuery, name, lenght)
for hit in iteration[4]:
#a hit is a resultat of blast, a hit in xml file
id = hit[1].text
queryDef = hit[2].text
log.debug(queryDef)
for hsp in hit[5]:
#a hsp is result of blast hit, like sequence or score… ; is hit_hsps°in xml
eValue = float(hsp.find('Hsp_evalue').text)
gaps = int(hsp.find('Hsp_gaps').text)
identity = int(hsp.find('Hsp_identity').text)
positive = int(hsp.find('Hsp_positive').text)
qSeq = hsp.find('Hsp_qseq').text
mSeq = hsp.find('Hsp_midline').text
hSeq = hsp.find('Hsp_hseq').text
hSeqLen = int(hsp.find('Hsp_align-len').text)
scores = {
'eValue': eValue,
'gaps': gaps,
'identity': identity,
'positive': positive
}
blast[numberQuery][id] = Hit(id, queryDef, scores, qSeq, hSeq,
mSeq, hSeqLen)
count += 1
log.info(str(len(blast)) + ' sequences was submited')
log.info(str(count) + ' sequences in total')
def _launch_and_validate_ib_gateway(self, retry=False, relaunch=False):
# _launcher.launch_ib_gateway_and_auth(retry_auth=self._connection_attempts > 1)
self._connection_attempts += 1
if not retry:
_launcher.launch_ib_gateway_and_auth()
elif relaunch:
_launcher.relaunch()
else:
res = requests.post(f"{API_URL}/iserver/reauthenticate", verify=False)
L.debug(res.content)
sleep(3)
self._call_post_auth_methods()
sleep(2)
self._validate_gateway_auth()
def get_next_node(self, node_data):
next_node_id = node_data.get('NextNodeId', '') #Fallback node id
variable_data = self.state.get("var_data", {})
buttons = node_data.get("Buttons")
for button in buttons:
root_key = re.split(r'\.|\[', button.get("ConditionMatchKey"))[0]
logger.debug(
f"Variable Data received for condition call is {variable_data}"
)
# if isinstance(variable_data, str):
# try:
# variable_data = json.loads(variable_data)
# except Exception as err:
# logger.error(f"Error parsing variable_data {variable_data}")
# variable_data = {}
logger.debug(
f"Variable Data after dict conversion is {variable_data}")
if variable_data.get(root_key) is None:
continue
path = button.get("ConditionMatchKey")
obj = {root_key: variable_data[root_key]}
variable_value = Util.deep_find(obj, path)
match_operator = button.get("ConditionOperator")
match_value = AnaHelper.verb_replacer(text=button.get(
"ConditionMatchValue", ""),
state=self.state)
logger.debug(
f"variable_value {variable_value} {variable_value.__class__} match_operator {match_operator} match_value {match_value}"
)
condition_matched = AnaHelper.is_condition_match(
variable_value, match_operator, match_value)
logger.debug(f"Condition matched is {condition_matched}")
if condition_matched:
variable_data = self.state.get("var_data", {})
node_variable_name = node_data.get("VariableName")
if node_variable_name:
button_variable_value = button.get("VariableValue")
button_variable_value = AnaHelper.verb_replacer(
text=button_variable_value, state=self.state)
variable_data[node_variable_name] = button_variable_value
next_node_id = button["NextNodeId"]
break
next_node_key = self.state.get("flow_id", "") + "." + next_node_id
node_data = AnaNode(next_node_key).get_contents()
return {"id": next_node_key, "data": node_data}
def sample(self, index_name=None, size=3):
"""
Provides a sample of documents from the index
Args:
index_name (string, optional) name of index, defaults to index name defined when initiating the class
size (int, optional) number of results to retrieve, defaults to 3, max 10k, can be relaxed with elastic config
"""
if not index_name:
if self.index_name:
index_name = self.index_name
else:
raise ValueError('index_name not provided')
res = self.es.search(index=index_name, size=size)
logger.debug(
f"Successfully sampled {len(res['hits']['hits'])} docs from {index_name}"
)
return res
def __process_repeatable_item(ana_repeatable, state, is_carousel):
variable_data = state.get("var_data", {})
repeat_on_varname = ana_repeatable.get("RepeatOn", None)
repeat_on = variable_data.get(repeat_on_varname, None)
if repeat_on is None:
logger.debug(f"No exact repeat_on_varname")
root_key = re.split(r"\.|\[", repeat_on_varname)[0]
if variable_data.get(root_key, None) is not None:
repeat_on = Util.deep_find({root_key:variable_data[root_key]}, repeat_on_varname)
logger.info("repeat_on: " + json.dumps(repeat_on))
repeat_as_varname = ana_repeatable.get("RepeatAs", None)
start_position = ana_repeatable.get("StartPosition", 0)
max_repeats = ana_repeatable.get("MaxRepeats", None)
end_position = None
if max_repeats:
end_position = start_position + max_repeats
pass
resulting_items = []
if isinstance(repeat_on, list):
for item in repeat_on[start_position:end_position]:
tempState = {}
tempState['var_data'] = copy.deepcopy(variable_data)
tempState['var_data'][repeat_as_varname] = item
item_json = json.dumps(ana_repeatable)
replaced_item_json = AnaHelper.verb_replacer(text=item_json, state=tempState)
replaced_item = json.loads(replaced_item_json)
if is_carousel:
car_buttons = replaced_item.get("Buttons", [])
car_buttons = AnaHelper.process_repeatable(car_buttons, tempState, False)
for car_button in car_buttons:
car_button["DoesRepeat"] = False
replaced_item["Buttons"] = car_buttons
pass
replaced_item['_id'] = replaced_item.get('_id', '') + "--" + str(repeat_on.index(item))
resulting_items.append(replaced_item)
pass
pass
pass
return resulting_items
