def _persist_data(self,
var_data,
session_id="",
channel="",
business_name="",
flow_id="",
business_id=""):
# change this method to perform async
if var_data == {}:
return 1
object_id = str(uuid.uuid4())
timestamp = datetime.datetime.utcnow()
collection = DB["user_data"]
document = {
"_id": object_id,
"user_id": self.user_id,
"flow_id": flow_id,
"session_id": session_id,
"business_id": business_id,
"data": var_data,
"channel": channel,
"business_name": business_name,
"timestamp": timestamp
}
try:
saved_document_id = collection.insert_one(document).inserted_id
logger.info(
f"Variable data saved with object_id {saved_document_id}")
return 1
except Exception as err:
logger.error(err)
raise
def cmdloop(self):
line = ''
args = []
stop = Command.SUCCESS
logger.debug(f"Start the command loop on class `{__class__.__name__}`")
self.preloop()
while stop != Command.EXIT:
try:
line = self.raw_input(self.prompt)
try:
args = self.parsecmd(line)
except ValueError as e:
logger.debug('Failed to parse the command line')
logger.error(str(e))
continue
args = self.precmd(args)
stop = self.cmdhandler(args)
stop = self.postcmd(stop, args)
except (KeyboardInterrupt, EOFError):
stop = self.interrupt_handler(args)
except BaseException as e:
stop = self.exception_handler(args, e)
self.postloop()
logger.debug(f"End the command loop on class `{__class__.__name__}`")
def run(self):
ret = self.operationWarmup()
if ret:
try:
self.subSytemDatabase.conn.commit()
except Exception as e:
logger.error(f"{e}")
logger.error(e, exc_info=True)
return
else:
self.signalConsole.emit(f"warmup complete, scrapping now...")
else:
self.signalConsole.emit(f"error: warmup failed, not scrapping")
return
ret = self.operationInit()
if ret:
self.signalConsole.emit(f"scrap completed, performing post process functions")
else:
self.signalConsole.emit(f"error: scrap process failed!")
return
self.subSytemScrap.driver.close()
def __get_next_node_id(cls, data, state, node_data):
next_node_id = node_data.get('NextNodeId', '') # Fallback node id
for button in node_data.get('Buttons', []):
try:
root_key = re.split(r'\.|\[',
button.get("ConditionMatchKey"))[0]
if data.get(root_key) is None:
data[root_key] = None
logger.debug("rootKey " + root_key)
path = button.get("ConditionMatchKey")
obj = {root_key: data[root_key]}
variable_value = Util.deep_find(obj, path)
match_operator = button.get("ConditionOperator")
match_value = AnaHelper.verb_replacer(text=button.get(
"ConditionMatchValue", ""),
state=state)
condition_matched = AnaHelper.is_condition_match(
variable_value, match_operator, match_value)
if condition_matched:
next_node_id = button["NextNodeId"]
break
except:
logger.error("error in btn " + str(button))
pass
return next_node_id
def send_messages(messages, sending_to):
endpoints = {"USER": application_config["GATEWAY_URL"], \
"AGENT": application_config["AGENT_URL"]}
url = endpoints.get(sending_to)
if url is None:
return 0
headers = {"Content-Type": "application/json"}
if messages == []:
logger.info(f"No messages to send to {sending_to}")
return 1
#This is deliberately synchronous to maintain order of messages being
#sent
for message in messages:
json_message = json.dumps(message)
try:
response = requests.post(url,
headers=headers,
data=json_message)
logger.info(response)
logger.info(f"Message sent to {sending_to} {message}")
except Exception as err:
logger.error(err)
return 0
return 1
def test_for_nans(data, name):
if not len(data):
raise ValueError('Data is empty!')
nan_rows = data[data.isnull().values.any(axis=1)]
if not len(nan_rows):
logger.info('\t\tNo Nans found in {}'.format(name))
return
logger.info(
'\t\tFound {} rows ({:.1f}%) and {} columns (out of {}) with Nans in {}'
.format(len(nan_rows), 100 * len(nan_rows) / len(data),
data.isnull().values.any(axis=0).sum(), len(data.columns),
name))
nan_counts = {}
for source, source_data in data.groupby('source'):
source_nans = {}
for column in source_data.columns:
col_nans = source_data[column].isnull().values.sum()
if col_nans:
source_nans[column] = col_nans
nan_counts[source] = source_nans
for k, v in nan_counts.items():
logger.error('\t\tFor source {}, found NaNs in {} columns:'.format(
k, len(v)))
for l, c in v.items():
logger.error('\t\t\t{:20} {:8} ({:.1f}%)'.format(
l, c, 100 * c / len(data)))
raise ValueError
def parse_settings(settings_file) -> dict:
""" Creates multiple dictionaries containing the settings parsed from a settings file.
Each type of plot has its own settings dictionary.
settings_file is the name of the text file containing the settings
Return values:
data is a pandas.DataFrame object which contains the alternative splicing data
hive_plot_settings is a dictionary containing the settings for the hive plot
struct_plot_settings is a dictionary containing the settings for the structure plot
"""
try:
config = configparser.ConfigParser()
logger.info('Reading settings from {0}...'.format(settings_file))
config.read(settings_file)
# hive_plot_settings = parse_hive_plot_settings(config)
# struct_plot_settings = parse_struct_plot_settings(config)
return parse_sashimi_settings(config)
# logger.error('Done reading settings.')
# return hive_plot_settings, struct_plot_settings, sashimi_plot_settings
except IOError:
logger.error('{0} is not a valid file path')
sys.exit(1)
def hook_start(self, session: WebshellSession):
'''当session创建成功时执行'''
super().hook_start(session)
p = PHPPayload('php/base/baseinfo.php')
ret = self.eval(p)
if not ret.is_success():
logger.error("Basic info gather failed!")
return
info = json.loads(ret.data)
for k, v in info.items():
if isinstance(v, str):
info[k] = base64.b64decode(v.encode()).decode(
self.options.encoding, 'ignore')
session.state['name'] = info['host']
session.state['pwd'] = info.get('pwd').strip()
session.state['description'] = self.help.lstrip('\r\n ').split('\n')[0]
session.server_info.lang = self.PHP
session.server_info.user = info.get('user').strip()
session.server_info.webshell_root = info.get('pwd').strip()
session.server_info.os_type = info.get('os_type').strip()
session.server_info.tmpdir = info.get('tmpdir').strip()
session.server_info.sep = info.get('sep').strip()
session.server_info.domain = info.get('domain')
session.server_info.group = info.get('group')
session.server_info.os_bits = info.get('os_bits')
def exception_handler(self, args: Cmdline, e: BaseException) -> int:
if isinstance(e, SystemExit):
return Command.STOP
elif isinstance(e, CommandNotFound):
logger.error(e)
return Command.STOP
return super().exception_handler(args, e)
def _payload(self, _____path:str, isstring=False, **kw)->Payload:
'''根据服务端语言构造payload
'''
if not isinstance(self.session, WebshellSession):
return None
payload = None
if self.session.server_info.lang == WebshellBase.PHP:
payload = PHPPayload(None, **kw)
elif self.session.server_info.lang == WebshellBase.ASP_NET_CS:
payload = CSharpPayload(None, **kw)
else:
payload = Payload(None, **kw)
if payload:
if isstring:
payload.from_string(_____path)
else:
path = os.path.join(os.path.dirname(call_path(2)), _____path)
try:
payload.from_file(path)
except FileNotFoundError:
logger.error(f"Cannot find file `{path}`, maybe the accessed payload is not implemented!")
raise SystemExit
return payload
def filter_album_tracks(oauth_token: str, recent_tracks: list) -> list:
try:
tracks = []
for track in recent_tracks:
album_tracks = get_album_tracks(oauth_token, track['album_id'])
df_album_tracks = pd.DataFrame(album_tracks)
df_album_tracks = df_album_tracks.filter(['items'])
df_album_tracks = pd.json_normalize(
df_album_tracks.to_dict('records'))
df_album_tracks = df_album_tracks.filter(
['items.id', 'items.track_number'])
df_album_tracks.rename(columns={
'items.id': 'track_id',
'items.track_number': 'track_number'
},
inplace=True)
df_album_tracks[
'track_number'] = df_album_tracks['track_number'] - 1
current_track = pd.merge(pd.DataFrame(recent_tracks),
df_album_tracks,
on='track_id',
how='inner').to_dict('records')[0]
tracks.append(current_track)
logger.info('Album tracks have been filtered successfully')
return tracks
except Exception as error:
logger.error(f'Error to filter album tracks: {error}')
raise
def get_dir_content(self, dir_path):
if dir_path == "/" or dir_path == "\\":
dir_path = ""
response = self.dbx.files_list_folder(dir_path)
entries = []
for entry in response.entries:
if type(entry) == dropbox.files.FolderMetadata:
entries.append(FolderInfo(entry.name, entry.path_display))
elif type(entry) == dropbox.files.FileMetadata:
entries.append(
FileInfo(entry.name, entry.size, entry.path_display))
else:
logger.error(
"DROPBOX : Unknown entry in dir {0}".format(dir_path))
while response.has_more:
response = self.dbx.files_list_folder_continue(response.cursor)
for entry in response.entries:
if type(entry) == dropbox.files.FolderMetadata:
entries.append(FolderInfo(entry.name, entry.path_display))
elif type(entry) == dropbox.files.FileMetadata:
entries.append(
FileInfo(entry.name, entry.size, entry.path_display))
else:
logger.error(
"DROPBOX : Unknown entry in dir {0}".format(dir_path))
return entries
def send_email(args):
logger.info("input parameter is {} {}".format(args.PLATFORM,
args.REPORTWEEKLY))
if args.REPORTWEEKLY:
weekly = args.REPORTWEEKLY
else:
weekly = "WW" + str(int(time.strftime("%W")) + 1)
if args.PLATFORM == "all":
platforms = allplatform
else:
platforms = args.PLATFORM.split(":")
logger.info("{} | {}".format(platforms, weekly))
for platform in platforms:
execlpath, emaillist = get_config(platform)
logger.info("{} | {}".format(execlpath, emaillist))
if execlpath:
execlcontent = read_execl(execlpath, weekly)
summaryinfo = analysis_reult(execlcontent)
print summaryinfo
template_dir = os.path.join(cur_dir, "email_template")
mail_title = "{0} Platform {1} Cycling Test Report".format(
platform, weekly)
#mail_sender = "*****@*****.**"
mail_sender = "*****@*****.**"
template_file = "template.html"
logser = r"\\ccr\ec\proj\deg\PID\BKC\Auto\ProductCyclingRecord"
if dirdict.has_key(platform):
print platform
logdir = os.path.join(logser, dirdict[platform])
else:
logdir = logser
print 'Generate mail content'
res_dict = {
"test_result_list": execlcontent,
"log_dir": logdir,
"summary_info": summaryinfo,
"platform": platform,
"weekly": weekly
}
env = Environment(loader=FileSystemLoader(template_dir))
template = env.get_template(template_file)
mail_body = template.render(res_dict)
logger.info('Send mail to {}'.format(emaillist))
attchment_list = []
if os.path.exists(execlpath):
attchment_list.append(execlpath)
if send_mail(mail_sender,
emaillist,
mail_title,
mail_body,
ccs="",
attachment=attchment_list):
print 'Mail has been sent successfully'
else:
print 'Failed to send mail'
else:
logger.error("Can not found the execl file")
def deep_find(obj, path):
try:
val = jsonpath(obj, path)
if bool(val) and isinstance(val, list) and len(val) == 1:
val = val[0]
return val
except Exception as err:
logger.error(err)
return None
def log_event(self, type_of_event, data):
logger_method = getattr(self, "log_%s" % type_of_event.lower(), None)
if logger_method is None:
logger.error(f"Unknown event type published {type_of_event}")
message_logged = logger_method(data)
return message_logged
def _task_status_from_pull_request(pull_request: PullRequest) -> str:
if not pull_request.closed():
return "Open"
elif pull_request.closed() and pull_request.merged():
return "Merged"
elif pull_request.closed() and not pull_request.merged():
return "Closed"
else:
logger.error("Pull request is in an invalid state")
return ""
def _automate_auth():
user_name, password, is_live = creds_conf["ib_user_name"], creds_conf[
"ib_password"], creds_conf["is_live_account"]
sms_receiver = MFASMSReceiver(60)
if is_live:
sms_receiver.start_listening_for_auth_code()
driver = webdriver.PhantomJS(service_args=['--ignore-ssl-errors=true'],
service_log_path="/tmp/phantom_logs.log")
driver.get("https://localhost:5000")
un_box = driver.find_element_by_id("user_name")
pw_box = driver.find_element_by_id("password")
submit_btn = driver.find_element_by_id("submitForm")
un_box.send_keys(user_name)
pw_box.send_keys(password)
submit_btn.click()
if is_live:
# IB uses MFA only for live accounts
SECURITY_CODE_BOX_ID = "chlginput"
try:
WebDriverWait(driver, 5).until(
EC.presence_of_element_located((By.ID, SECURITY_CODE_BOX_ID)))
except TimeoutException:
logger.error(
"IB auth failed: MFA security code element wasn't present")
driver.quit()
raise
auth_code = sms_receiver.auth_code
if auth_code is not None:
sec_code_box = driver.find_element_by_id(SECURITY_CODE_BOX_ID)
sec_code_box.send_keys(auth_code)
submit_btn = driver.find_element_by_id("submitForm")
submit_btn.click()
else:
raise SystemError("IB auth code for MFA was not received")
try:
WebDriverWait(driver, 10).until(
EC.text_to_be_present_in_element((By.CSS_SELECTOR, "pre"),
LOGIN_SUCCEEDS_PHRASE))
except TimeoutException:
logger.error(
f"Login to IB failed, success page loading timed out, page source: {driver.page_source}"
)
raise
finally:
driver.quit()
return True
def cmdhandler(self, args: Cmdline)-> int:
if args.cmd is None:
return Command.STOP
cmd = args.cmd
func = self.session.command_map.get(cmd)
if func is None:
logger.error(f'No command named `{cmd}`.')
return Command.STOP
ret = func.run(args)
logger.debug(f'A command line `{args.cmdline}` is executed, returns `{ret}`')
return ret
def normalize(self, url):
if url.endswith('.'):
url = url[:-1]
try:
session = requests.Session()
resp = session.head(url, allow_redirects=True, timeout=5)
unshorten_url = resp.url
return url_normalize(unshorten_url)
except requests.exceptions.RequestException as e:
logger.error('Error {e} {url}'.format(e=e, url=url))
return INVALID_URL
return INVALID_URL
def filter_user_playback(oauth_token: str, recent_tracks: list) -> None:
try:
for recent_track in reversed(recent_tracks):
status_code = start_user_playback(oauth_token,
recent_track['album_id'],
recent_track['track_number'])
if status_code == 204:
time.sleep(7)
logger.info(f'Recent tracks have been played successfully')
except Exception as error:
logger.error(f'Error to filter user playback: {error}')
raise
def main():
load_configuration()
logger.info("Private Cloud started")
while True:
try:
dispatch_remote_iteration(g.dropbox_client)
dispatch_local_iteration(g.dropbox_client)
logger.info("Finish iteration. sleeping...")
time.sleep(g.sync_interval_in_sec)
except Exception as e:
logger.error("Exception occurred: {0}".format(str(e)))
def __init__(self,
response: requests.Response,
hint: str = None,
description: str = None,
message: str = None):
super().__init__(response.status_code, hint, description, message)
logger.error(
f"[PostgREST] Error {response.status_code} - {response.url}")
self.response = response
# NOTE: Maybe this is bad? We are redifining error_body with a new value
self.error_body = self.postgres_error_details(self.response,
self.error_body)
def message_handler():
message = request.get_json()
logger.info(f"Message Received {message}")
handle_message = MessageHandlerPool.submit(
MessageProcessor(message).respond_to_message)
exception = handle_message.exception()
if exception:
logger.error(exception)
return jsonify(status="received")
def _make_dataframe(self):
file_data = self._bucket.get_response_body(self._filename)
try:
dataframe: pd.DataFrame = pd.read_json(file_data)
except Exception:
logger.error(f'ERROR {self._filename}')
# pandas having trouble with opening file bce5476a-09e4-4e44-a3cc-eca0090a106c, and throws protocol error
# but python json module deals with it nice
file_data = json.loads(file_data)
dataframe = pd.DataFrame(file_data)
return dataframe
def CreateConnection(self) -> bool:
try:
self.socket = Socket(AF_INET, SOCK_DGRAM)
except Exception as e:
logger.error(f"could not create socket\n{e}")
return False
else:
logger.info("binding socket")
finally:
# set socket receive buffer to 1 packet to avoid buffer bloat and packet accumulation
self.socket.setsockopt(SOL_SOCKET, SO_RCVBUF, UDP_MSGLEN)
self.socket.settimeout(self.timeout)
return True
def configure_telegram():
"""
Configures the bot with a Telegram Token.
Returns a bot instance.
"""
TELEGRAM_TOKEN = os.environ.get('TELEGRAM_TOKEN')
if not TELEGRAM_TOKEN:
logger.error('The TELEGRAM_TOKEN must be set')
raise NotImplementedError
return telegram.Bot(TELEGRAM_TOKEN)
def handle_set_session_data(self, event):
data = event.get("data", "{}")
try:
dict_data = json.loads(data)
var_data = self.state.get("var_data", {})
final_var_data = Util.merge_dicts(var_data, dict_data)
self.state["var_data"] = final_var_data
except ValueError:
logger.error(f"Set session data payload is not in json format {data}")
return []
def __construct_user_messages(cls, input_type, data):
messages_data = []
if input_type == "TEXT":
message_type = MessageType.get_value("SIMPLE")
text = data.get("val", "")
message_content = MessageContent(text=text, mandatory=1).trim()
message_data = MessageData(type=message_type,
content=message_content).trim()
else:
logger.error(f"Unsupported input_type for agent {input_type}")
messages_data.append(message_data)
return messages_data
def get_contents(self):
response = CACHE.get(self.node_key)
if response is None:
logger.warning(f"Data not found for {self.node_key}")
return {}
try:
response_dict = json.loads(response)
return response_dict
except Exception as err:
logger.error(err)
raise
def dispatch_remote_iteration(client):
"""
Walk on remote cloud files:
* Create dir if exist only on remote
* If file name start with upload magic, upload from local to remote (if configured)
* If file name start with download magic, download from remote to local (if configured)
:param client: ClientBase
"""
logger.info("dispatch_remote_iteration started")
remote_dir = client.get_dir_content('/')
while len(remote_dir) != 0:
current = remote_dir.pop()
if current.__class__ == FolderInfo:
remote_dir += client.get_dir_content(current.folder_path)
if not os.path.exists(os.path.advjoin(g.local_cloud_path, current.folder_path)):
os.mkdir(os.path.advjoin(g.local_cloud_path, current.folder_path))
elif not os.path.isdir(os.path.advjoin(g.local_cloud_path, current.folder_path)):
logger.error("dispatch_remote_iteration : remote path is folder, and on local its a file")
else:
if g.enable_upload and current.file_name.startswith(g.upload_prefix):
logger.info("dispatch_remote_iteration : found file name start with upload magic,"
" uploading from local cloud")
local_file = get_local_path_without_prefix(g.upload_prefix, g.local_cloud_path,
current.file_path, current.file_name)
remote_file = get_remote_path_without_prefix(g.upload_prefix, current.file_path, current.file_name)
if not os.path.exists(local_file):
logger.error("dispatch_remote_iteration : "
"remote file start with prefix and there is no local file")
client.delete_file(current.file_path)
client.upload_file(local_file, remote_file)
elif g.enable_download and current.file_name.startswith(g.download_prefix):
logger.info("dispatch_remote_iteration : found file name start with download magic,"
" download to local cloud and leave thin version on remote cloud")
local_path = get_local_path_without_prefix(g.download_prefix, g.local_cloud_path,
current.file_path, current.file_name)
remote_path = get_remote_path_without_prefix(g.download_prefix, current.file_path, current.file_name)
partial_file_path = os.path.advjoin(g.tmp_path, os.path.basename(remote_path))
client.download_file(current.file_path, local_path)
client.delete_file(current.file_path)
create_partial_file(local_path, g.thin_mode_byte_length, partial_file_path)
client.upload_file(partial_file_path, remote_path)
os.remove(partial_file_path)
elif not os.path.exists(os.path.advjoin(g.local_cloud_path, current.file_path)) \
and not_start_with_saved_prefix(current.file_name):
logger.info("dispatch_remote_iteration : found file that only on remote cloud, downloading it")
client.download_file(current.file_path, os.path.advjoin(g.local_cloud_path, current.file_path))
