def third_user_login(request):
"""
The method is third user login
@param request: third_key, third_type
@return:
"""
if request.method == 'GET':
return response_fail_to_mobile(500, REQUEST_METHOD_ERROR)
try:
username = request.POST['third_key']
password = request.POST['third_type']
user = authenticate(username=username, password=password, type=3)
auth_login(request, user)
logic = UserProfileLogic.get_user_profile_by_user_id(user_id=request.user.id)
last_login = datetime_convert_current_timezone(logic.last_login_time)
if last_login is not None:
last_login = datetime_to_string(last_login)
if logic.user_nick is None:
user_nick = ''
else:
user_nick = logic.user_nick
rec = dict()
rec['session_id'] = request.session._session_key
rec['user_id'] = request.user.id
rec['user_name'] = user_nick
rec['last_login'] = last_login
UserProfileLogic.update_last_login_time(request.user.id)
return response_success_to_mobile(rec)
except Exception as e:
error(e)
auth_logout(request)
return response_fail_to_mobile(500, 'Third part account login fail!')
def wechat_recv_param_wapper(self, *args, **kwargs):
if not signature_checker(token,
kwargs['timestamp'],
kwargs['nonce'],
kwargs['signature']):
logger.error("wechat_recv_wapper, failed, access_token:%s args:%s, kwargs:%s" %
(token, args, kwargs))
return "-1"
# 如果没有内容尽是验证，则即可返回
if not self.request.body:
return kwargs.get("echostr", "0")
# 内容解析
crypt = WXBizMsgCrypt(token, encodingAESKey, appid)
ret, xml_body = crypt.DecryptMsg(self.request.body,
kwargs['msg_signature'],
kwargs['timestamp'],
kwargs['nonce'])
assert ret == WXBizMsgCrypt_OK
order_dic_body = xmltodict.parse(xml_body)
dic_body = dict(order_dic_body["xml"])
kwargs['body'] = dic_body
logger.info("%s:wechat_recv_wapper args:%s kwargs:%s" % (fun.__name__, args, kwargs))
fun(self, *args, **kwargs)
return kwargs.get("echostr", "0")
def find_services(self, service_type, rdm_type=RT_CPU_USAGE_RDM, rdm_param=1):
"""
查找服务列表
:param service_type:服务类型
:param rdm_type:随机类型，0选择cpu使用率最低的；1一致性hash选择
:param rdm_param:如果随机类型是0,参数整形,表示随机个数
如果随机类型是1,list形式,hash key 列表
:return:
"""
if rdm_type == RT_HASH_RING and not isinstance(rdm_param, list):
rdm_param = [rdm_param]
if not ServiceGrpMgr().get_service_grp(service_type):
logger.error("TcpHandler::find_service Failed!!!, ERROR_PARAMS_ERROR, service_type:%s" % service_type)
return {}
service_obj_ls = ServiceMgr().get_run_services(service_type, rdm_type, rdm_param)
if not service_obj_ls:
return {}
result = {}
for idx, service_obj in enumerate(service_obj_ls):
result_key = idx if rdm_type == RT_CPU_USAGE_RDM else rdm_param[idx]
result[result_key] = {"ip": service_obj.ip,
"port": service_obj.port,
"jid": service_obj.jid}
return result
def _get_file_data(self, data_dictionary, repo):
""" Create a dictionary that contains name, uri, hash, size, and
pkg_type.
file_data format:
"file_data":[
{ "file_name" : name of package
"file_uri" : uri here
"file_hash" : sha256
"file_size" : size in kb's
## "pkg_type" : primary or dependency
}
]
"""
# TODO: implement a for loop for the possibility of multiple uri's
file_data = []
# uri
uri = data_dictionary.get(PkgDictValues.uri, '')
if uri:
hostname = repo.split(' ')[0]
uri = hostname + data_dictionary[PkgDictValues.uri]
# name
name = uri.split('/')[-1]
# hash
pkg_hash = data_dictionary.get(PkgDictValues.sha256, '')
if not pkg_hash:
pkg_hash = data_dictionary.get(PkgDictValues.sha1, '')
if not pkg_hash:
pkg_hash = data_dictionary.get(PkgDictValues.md5, '')
# size
size = data_dictionary.get('Size', '')
try:
size = int(size)
except Exception as e:
logger.error("Failed to cast file_size to int.")
logger.exception(e)
# If for whatever reason it fails to convert
size = ''
# Package is marked as primary, the dependencies added
# to file_data are marked as dependency.
#pkg_type = 'primary'
file_data = [{FileDataKeys.name: name,
FileDataKeys.uri: uri,
FileDataKeys.hash: pkg_hash,
FileDataKeys.size: size}]
#'pkg_type': pkg_type}]
return file_data
def _dpkg_install(self, update_dir=None):
"""Install an update or install a new package.
If an update directory is given, it must be made sure that
the directory contains all of the dependencies for ALL of the
packages inside the directory.
"""
if update_dir:
install_command = ['dpkg', '-i', '-R', update_dir]
try:
result, err = self.utilcmds.run_command(install_command)
if err:
raise Exception(err)
logger.info(
('debhandler.py/_dpkg_install: '
'Installed all packages in: ') + update_dir)
return 'true', ''
except Exception as e:
logger.error(
('debhandler.py/_dpkg_install: '
'Failed to install packages in: ') + update_dir)
return 'false', str(e)
else:
logger.info(
'debhandler.py/_dpkg_install: No directory provided.')
return 'false', 'Update dir: ' + update_dir
def get(self):
"""
Attempts to get an operation from the queue if no operation is pending.
Returns:
The operation if it was successfully retrieved, None otherwise.
"""
operation = None
if (not self.op_in_progress) and (not self.paused):
try:
operation = self.queue.get_nowait()
self.op_in_progress = True
try:
logger.debug(
"Popping {0} from OpQueue.".format(operation.id)
)
except Exception:
logger.debug("Popping {0} from OpQueue.".format(operation))
except Queue.Empty as e:
# logger.debug("Operations queue is empty.")
operation = None
except Exception as e:
logger.error("Error accessing operation queue.")
logger.error("Message: %s" % e)
operation = None
return operation
def get_block(self, block_hash):
block = self.bitcoind('getblock', (block_hash,))
rawtxreq = []
i = 0
for txid in block['tx']:
rawtxreq.append({
"method": "getrawtransaction",
"params": (txid,),
"id": i,
})
i += 1
postdata = dumps(rawtxreq)
while True:
try:
response = urllib.urlopen(self.bitcoind_url, postdata)
r = load(response)
response.close()
except:
logger.error("bitcoind error (getfullblock)")
self.wait_on_bitcoind()
continue
try:
rawtxdata = []
for ir in r:
assert ir['error'] is None, "Error: make sure you run bitcoind with txindex=1; use -reindex if needed."
rawtxdata.append(ir['result'])
except BaseException as e:
logger.error(str(e))
self.wait_on_bitcoind()
continue
block['tx'] = rawtxdata
return block
def do_establish_taskqueue(self, tqname=None, namespace='mobile', account=1, version=1, description=None):
'''
Establish one taskqueue. If the task queue exists, the establishment would been cancelled.
Params:
tqname: Name of task queue.
namespace: Namespace of the task queue.
account: Account ID.
version: Version of the resource, default value 1.
description: Description for the task queue
Return:
True: Establish success.
False: Establish failed.
'''
assert(tqname)
resource = TQRES_TEMPLATE_STR % (version, account, namespace, tqname)
logger.debug('Establish one taskqueue - name:%s, resource:%s, description:%s' % (tqname, resource, description))
result = self.do_get_taskqueue(tqname, namespace, account, version)
if result == True:
logger.debug('Task Queue exists, canceling the establishment!')
return True
else:
if isinstance(description, basestring):
description = {'description' : description}
elif isinstance(description, dict):
pass
else:
logger.error('Establish Task Queue failed! Unknow description type!')
return False
data = json.dumps(description)
result = self.m_wrest.do_access(resource, 'PUT', data=data, headers=None)
if result.code >= 200 and result.code < 300:
logger.debug('Establish Task Queue success!')
return True
def _parse_packages_to_install(self, data):
lines = data.split('\n')
install_list = []
for line in lines:
if 'Inst' in line:
try:
#package name comes right after Inst
#Example: "Inst {pkg-name} .....other data....."
update_package_name = line.split(' ')[1]
# New version comes right after old version
#Example: "Inst {name} [current-version] (new-version ....)
get_version_regex = r'.*\((\S+).*\)'
update_package_version = \
re.match(get_version_regex, line).group(1)
install_pkg = (update_package_name, update_package_version)
install_list.append(install_pkg)
except AttributeError as e:
logger.error(
'Failed retrieving version for: ' + update_package_name
)
logger.exception(e)
return install_list
def _apt_install(self, package_name):
logger.debug('Installing {0}'.format(package_name))
install_command = [self.APT_GET_EXE, 'install', '-y', package_name]
#TODO: figure out if restart needed
restart = 'false'
# TODO: parse out the error, if any
try:
result, err = self.utilcmds.run_command(install_command)
if err:
# Catch non-error related messages
if 'reading changelogs' in err.lower():
pass
else:
raise Exception(err)
except Exception as e:
logger.error('Faled to install {0}'.format(package_name))
logger.exception(e)
return 'false', str(e), restart
logger.debug('Done installing {0}'.format(package_name))
return 'true', '', restart
def complete_softwareupdate(self, install_data):
"""
Removes the product key directory if it exists, and lets
softwareupdate download and install on its own.
"""
success = "false"
error = "Failed to install: " + install_data.name
try:
app_plist_data = self._get_app_plist_data(install_data)
for i in range(1, 3):
remove_success = self._remove_productkey_dir(app_plist_data)
if remove_success:
break
time.sleep(5 * i)
update_name = self._get_softwareupdate_name(app_plist_data)
success, error = self.softwareupdate(update_name, install_data.proc_niceness)
except Exception as e:
logger.error("Failed to download/install pkg with softwareupdate: %s" % install_data.name)
logger.exception(e)
return success, error
def _post(url, data=None, json=None, etag=None, **kwargs):
try:
res = requests.post(url, data=data, json=json, **kwargs)
except requests.exceptions.RequestException as e:
logger.error(">> [POST] {0} failed with exception: {1}".format(url, e))
return None, e
return _rest_check_response(res)
def _is_boot_up(self):
"""Checks whether if the agent is coming up because of a reboot or not.
Returns:
(bool) True if system boot up detected, False otherwise.
"""
current_uptime = systeminfo.uptime()
boot_up = 'no'
try:
if os.path.exists(self._uptime_file):
with open(self._uptime_file, 'r') as f:
file_uptime = f.read()
if current_uptime < long(file_uptime):
boot_up = 'yes'
except Exception as e:
logger.error("Could not verify system bootup.")
logger.exception(e)
return boot_up
def _delete(url, **kwargs):
try:
res = requests.delete(url, **kwargs)
except requests.exceptions.RequestException as e:
logger.error(">> [DELETE] {0} failed with exception: {1}".format(url, e))
return None, e
return _rest_check_response(res)
def send_sms(cls, msg, level):
if level not in ('error', 'fatal'):
raise ValueError("level MUST be error or fatal")
to_ls = cls.error_s_ls if level == 'error' else cls.fatal_s_ls
result = []
for mobile in to_ls:
http_client = HttpRpcClient()
headers = {
'Accept': 'application/json',
'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8'
}
msg = {
'apiKey': API_KEY,
'appId': APP_ID,
'templateId': TEMPLATE['LOGGER'],
'mobile': mobile,
'param': msg
}
body = ujson.dumps(msg)
response = ujson.loads(http_client.fetch_async(URL, headers, body))
if response['result'] != 'SUCCESS':
logger.error('SMSService error_code: %s at %s' % (response['reason'], time.ctime()))
result.append(response)
return result
def load_queue(file_path):
if not os.path.exists(file_path):
return OperationQueue()
loaded = []
try:
with open(file_path, 'r') as _file:
loaded = cPickle.load(_file)
except Exception as e:
logger.error("Failed to load operations from: {0}".format(file_path))
logger.exception(e)
loaded = []
logger.debug("Loaded operations: {0}".format(loaded))
q = OperationQueue()
for operaion in loaded:
q.put(operaion)
return q
def start_service(self):
"""
启动服务
:return:
"""
args = ArgumentParser().args
try:
Jobs().add_interval_job(UPDATE_INTERVAL, self.update)
if not self.is_sm:
port = {"tcp": args.tcp_port}
port.update({"https": args.http_port} if args.is_https else {"http": args.http_port})
self.adv = ServiceAdvertiser(self.service_type, port, self.get_jid(), self.service_version)
self.adv.advertise()
checker_ls = []
self.add_port_checker(args, checker_ls)
PortChecker(checker_ls).start()
self.services(args, self.thread_ls)
self.__sys_services(args, self.thread_ls)
logger.warn("start services for %s, args:%s" % (self.service_type, args))
gevent.joinall([thread.start() for thread in self.thread_ls])
except:
logger.error(traceback.format_exc())
sys.exit(0)
def getfullblock(self, block_hash):
block = self.bitcoind('getblock', [block_hash])
rawtxreq = []
i = 0
for txid in block['tx']:
rawtxreq.append({
"method": "getrawtransaction",
"params": [txid],
"id": i,
})
i += 1
postdata = dumps(rawtxreq)
try:
respdata = urllib.urlopen(self.bitcoind_url, postdata).read()
except:
logger.error("groestlcoind error (getfullblock)",exc_info=True)
self.shared.stop()
r = loads(respdata)
rawtxdata = []
for ir in r:
if ir['error'] is not None:
self.shared.stop()
print_log("Error: make sure you run groestlcoind with txindex=1; use -reindex if needed.")
raise BaseException(ir['error'])
rawtxdata.append(ir['result'])
block['tx'] = rawtxdata
return block
def run(self):
while self.processor.shared.paused():
time.sleep(1)
self.ircname = self.host + ' ' + self.getname()
logger.info("joining IRC")
while not self.processor.shared.stopped():
client = irc.client.IRC()
try:
c = client.server().connect('irc.freenode.net', 6667, self.nick, self.password, ircname=self.ircname)
except irc.client.ServerConnectionError:
logger.error('irc', exc_info=True)
time.sleep(10)
continue
c.add_global_handler("welcome", self.on_connect)
c.add_global_handler("join", self.on_join)
c.add_global_handler("quit", self.on_quit)
c.add_global_handler("kick", self.on_kick)
c.add_global_handler("whoreply", self.on_who)
c.add_global_handler("namreply", self.on_name)
c.add_global_handler("disconnect", self.on_disconnect)
c.set_keepalive(60)
self.connection = c
try:
client.process_forever()
except BaseException as e:
logger.error('irc', exc_info=True)
time.sleep(10)
continue
logger.info("quitting IRC")
def do_get_task_total(self, tqname=None, namespace='mobile', account=1, version=1, tags=None):
'''
Get total number of the specific task queue.
Params:
tqname: Name of task queue.
namespace: Namespace of the task queue.
account: Account ID.
version: Version of the resource, default value 1.
tags: Search conditions.
Return:
Total number of tasks on this taskqueue
Exception message
'''
assert(tqname)
resource = TQRES_TEMPLATE_STR % (version, account, namespace, tqname)
resource = '/'.join([resource, 'stats'])
if tags:
data = json.dumps({'tags' : tags}, encoding="utf-8")
else:
data = tags
logger.debug('Get total tasks number - resource:%s, data:%s' % (resource, data))
result = self.m_wrest.do_access(resource, 'POST', data=data, headers=None)
if result.code >= 200 and result.code < 300:
logger.debug('Get Task Queue stats success!')
try: #parse stat data
tqs = json.loads(result.content)
total_tasks = tqs.get('total_tasks')
return total_tasks
except Exception, e:
logger.error('Get Task Queue stats Failed! %s' % repr(e))
def add_to_result_queue(self, result_operation, retry=True):
"""
Adds an operation to the result queue which sends it off to the server.
Arguments:
result_operation
An operation which must have a raw_result, urn_response,
and request_method attribute.
retry
Determines if the result queue should continue attempting to send
the operation to the server in case of a non 200 response.
"""
try:
if not isinstance(result_operation, ResultOperation):
result_operation = ResultOperation(result_operation, retry)
return self._result_queue.put(result_operation)
except Exception as e:
logger.error("Failed to add result to queue.")
logger.exception(e)
def format_tq_payload(self, source=None, encode=False, **keyval):
'''
Format payload members for task queue operation commands.
Params:
source : Input payload dict. If this param is empty, the interface would create a new one.
encode : Enable/Disable encoding. Default value is false. It would remove '\n' when enable encoding.
Converts data to json str at first, then with base64 encoding.
keyval : Key name and value of the specific payload.
Return:
dict of the payload
Str with base64 encoded.
Exception message
'''
if source:
assert(isinstance(source, dict))
retval = source
else:
retval = {}
retval.update(keyval) # append key
if encode:
# convert into json
try:
retval = json.dumps(retval)
except Exception, e:
logger.error('Encoding payload error: %s!' % e)
raise Exception(e)
# base64
retval = base64.encodingstring(retval).replace("\n", '')
def result_queue_file_dump(self):
try:
queuesave.save_result_queue(self._result_queue)
except Exception as e:
logger.error("Failed to save result queue to file.")
logger.exception(e)
def _move_pkgs(self, install_data, app_plist_data):
""" Move all pkgs in src to dest. """
try:
product_key = app_plist_data["productKey"]
src = os.path.join(settings.UpdatesDirectory, install_data.id)
dest = os.path.join("/Library/Updates", product_key)
if not os.path.exists(dest):
self._make_dir(dest)
time.sleep(3)
for _file in os.listdir(src):
if _file.endswith(".pkg"):
su_pkg_path = os.path.join(dest, _file)
if os.path.exists(su_pkg_path):
os.remove(su_pkg_path)
logger.debug("Removed existing pkg from /Library/Updates: %s " % su_pkg_path)
src_pkg = os.path.join(src, _file)
shutil.move(src_pkg, dest)
logger.debug("Moved " + _file + " to: " + dest)
except Exception as e:
logger.error("Failed moving pkgs to /Library/Updates.")
logger.exception(e)
raise
def main():
"""
The function gets all the user callback objects whose notify times are between the
current time and 1 minute from now. For each callback object it calls the function notification_callback,
each call is spawned as separate thread. This way, all users get concurrent notifications and if there
are a large number of notifications we won't be taking up more than 1 minute to complete the execution.
All the spawned threads are joined with this function, so that it waits until they complete their
execution.
@params: None
@output: None
"""
now = datetime.datetime.now()
now_time = datetime.time(now.hour, now.minute, now.second)
xmins_frm_now = now + datetime.timedelta(seconds = 60)
xmins_frm_now = datetime.time(xmins_frm_now.hour, xmins_frm_now.minute, xmins_frm_now.second)
today = datetime.date.today()
days = {0:'mon', 1:'tue', 2:'wed', 3:'thu', 4:'fri', 5:'sat', 6:'sun'}
today = days[today.weekday()]
user_callbacks = models.UserCallback.objects.filter(notify_time__gte=now_time, notify_time__lt=xmins_frm_now)
logger.info("number of user callbacks to process: %d" % user_callbacks.count())
threads = []
for user_callback in user_callbacks:
thread = threading.Thread(target = notification_callback, args = (user_callback, today,))
threads.append(thread)
try:
thread.start()
except Exception, e:
logger.error("error while running user callback: %s" % e)
def get_application(self, vendor_id):
""" Returns an Application instance based on the 'vendor_id'.
@param vendor_id: Vendor ID of the application.
@return: An Application instance.
"""
try:
with self._connection:
cursor = self._connection.cursor()
select_sql = "SELECT * FROM %s WHERE %s='%s'" % (
self._application_table, ApplicationColumn.VendorId,
vendor_id)
cursor.execute(select_sql)
row = cursor.fetchone()
app = self._get_application_from_row(row)
except Exception as e:
logger.error("Could not get application id %s." % vendor_id)
logger.exception(e)
app = None
return app
def run(self):
logger.info('Devices\' information sync with airwatch started at %s under account %d!' % (str(time.time()), self.aw_account))
try:
logger.info('Step 1: Get device information from RS.')
rs_retval = self._get_devinfo_from_rs()
logger.debug('Account %d includes %d device needed sync with airwatch!' % (self.aw_account, rs_retval))
if rs_retval <= 0:
# report
return # exit, no work to do
logger.info('Step 2: Get device information from airwatch.')
aw_retval = self._get_devinfo_from_aw(self.devinfo_original_queue, self.devinfo_refresh_queue)
logger.debug('Account %d, get %d device information from airwatch!' % (self.aw_account, aw_retval))
if (aw_retval != rs_retval):
#The devices do not exist in airwatch needed to be updated as unenroll status.
logger.warning('Account %d, Original device number (%d) and refresh device number (%d) NOT match!' % (self.aw_account, rs_retval, aw_retval))
logger.info('Step 3: Sync device information to RS.')
sync_retval = self._sync_devinfo()
logger.debug('Account %d, sync %d device information with RS finished!' % (self.aw_account, sync_retval))
if (aw_retval != sync_retval):
#causes of updating RS fail
logger.warning('Account %d, Refresh device number (%d) and Sync device number (%d) NOT match!' % (self.aw_account, aw_retval, sync_retval))
# Step 4: Update the defect devices to be unenroll.
if self.m_defect_devs:
defect_cnt = len(self.m_defect_devs)
logger.info('Step 4: Set %d devices to be "unenroll" status!' % defect_cnt)
ok_cnt = self._unenroll_dev_status(self.m_defect_devs)
if defect_cnt != ok_cnt:
logger.warning('Account %d, Set %d devices to be "unenroll status failed!"' % (self.aw_account, (defect_cnt - ok_cnt)))
# Step 5: Report
except Exception, e:
logger.error(repr(e))
def current_filesystem_data(self):
fs_list = []
try:
fs_data = self.filesystem_usage()
for fs in fs_data:
fs_dict = {}
fs_dict[MonitKey.Name] = fs[0]
fs_dict[MonitKey.Used] = fs[1]
fs_dict[MonitKey.PercentUsed] = fs[2]
fs_dict[MonitKey.Free] = fs[3]
fs_dict[MonitKey.PercentFree] = fs[4]
fs_dict[MonitKey.Mount] = fs[5]
fs_list.append(fs_dict.copy())
except Exception as e:
logger.error("Could not retrieve mac file system data.")
logger.exception(e)
return fs_list
def _apt_purge(self, package_name):
purged = 0
purge_command = [self.APT_GET_EXE, 'purge', '-y', package_name]
try:
result, err = self.utilcmds.run_command(purge_command)
if err:
raise Exception(err)
found = re.search('\\d+ to remove', result)
if found:
amount_purged = found.group().split(' ')[0]
purged = int(amount_purged)
if purged > 0:
logger.debug(
'Successfuly removed {0} packages.'.format(purged)
)
return 'true', ''
else:
logger.info('No packages were removed.')
return 'false', 'No packages removed.'
except Exception as e:
logger.error('Problem while uninstalling package: ' + package_name)
logger.exception(e)
return 'false', str(e)
def _get(url, params=None, etag=None, **kwargs):
try:
res = requests.get(url, params=params, **kwargs)
except requests.exceptions.RequestException as e:
logger.error(">> [GET] {0} failed with exception: {1}".format(url, e))
return None, e
return _rest_check_response(res)
def handle_message(self, from_jid, subject, body):
"""
消息接收分发
:param from_jid: 发送者jid
:param subject: 发送主题
:param body: 发送内容
:return:
"""
if subject not in self.subject_to_fun_dic:
error_msg = "subject:%s not register, details:%s" % (subject, self.subject_to_fun_dic)
logger.error(error_msg)
return True
self.subject_to_fun_dic[subject](self, from_jid, body)
return True
def startWorking(self,frame,box):
if frame is None:
self._isWorking = False
logger.error('追踪器追踪区域锁定失败')
return
c,r,w,h = box
roi = frame[r:r+h, c:c+w]
self.track_window = (c,r,w,h)
self.hsv_roi = cv2.cvtColor(roi, cv2.COLOR_BGR2HSV)
mask = cv2.inRange(self.hsv_roi, np.array((0., 30.,32.)), np.array((180.,255.,255.)))
self.roi_hist = cv2.calcHist([self.hsv_roi], [0], mask, [180], [0, 180])
cv2.normalize(self.roi_hist, self.roi_hist, 0, 255, cv2.NORM_MINMAX)
self.term_crit = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 80, 10)
self._isWorking = True
logger.info("追踪器开始工作")
def parse_configuration_file(self):
conf = ConfigParser.SafeConfigParser()
try:
if not conf.read(CRONJOB_CONFIG_FILE):
logger.info('Can not read %s' % CRONJOB_CONFIG_FILE)
return None, None
retrieve_date = conf.get('mdmi-updatevpnprofile',
'cronjob_retrieve_device_date')
threshold_num = string.atoi(
conf.get('mdmi-updatevpnprofile', 'cronjob_threshold_num'))
self.g_threshold_num = threshold_num
return retrieve_date, threshold_num
except Exception, e:
logger.error('parse configuration file error %s' % e)
return None, None
def list(itemtype):
""" List configured items (servers or packages)
"""
lookup = None
if (itemtype == 'server') or (itemtype == 'servers'):
lookup = 'vro_servers'
if (itemtype == 'package') or (itemtype == 'packages'):
lookup = 'packages'
if not lookup:
logger.error(""" Invalid itemtype to list from configuration:
only 'server'(s)/'package'(s) values are accepted.""")
exit(-1)
click.echo("%s configured items are:" % lookup)
for item in config[lookup]:
click.echo(" > %s" % str(item))
def load_file(filename: str) -> Any:
"""
Loads an object from a file using pickle
:param filename: a filename relevant to AGENT_PATH
:return: the object
"""
try:
file_name = os.path.join(AGENTS_PATH, filename)
with open(file_name, 'rb') as file:
unserialized_data = pickle.load(file)
except Exception as e:
logger.error(AGENT_UTILS_NAME, f'Error while reading from file {filename}. Error {e}')
return None
return unserialized_data
async def error_middleware(request, handler):
try:
resp = await handler(request)
import ipdb
ipdb.set_trace()
if isinstance(resp, dict):
return wrap_json(resp)
return resp
except Exception as e:
logger.error(e)
if e.status == 500:
return wrap_json({}, errcode=5000)
return wrap_json({}, errcode=4004)
def myDomainEventHandler(conn, dom, *args, **kwargs):
try:
vm_name = dom.name()
# print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
# print vm_name
# print kwargs
# support vnc
# if kwargs.has_key('detail') and kwargs.has_key('event'):
# event = str(DOM_EVENTS[kwargs['event']])
# detail = str(DOM_EVENTS[kwargs['event']][kwargs['detail']])
# logger.debug(event+" "+detail)
# if event == 'Started' and detail == 'Booted':
# modify_token(vm_name, 'Started')
# elif event == 'Stopped' and detail == 'Destroyed':
# modify_token(vm_name, 'Stopped')
if kwargs.has_key('event') and kwargs.has_key('detail') and \
str(DOM_EVENTS[kwargs['event']]) == "Undefined" and \
str(DOM_EVENTS[kwargs['event']][kwargs['detail']]) == "Removed":
try:
logger.debug('Callback domain deletion to virtlet')
# deleteVM(vm_name, V1DeleteOptions())
file_path = '%s/%s-*' % (DEFAULT_DEVICE_DIR, vm_name)
cmd = 'mv -f %s /tmp' % file_path
logger.debug(cmd)
runCmd(cmd)
except:
logger.error('Oops! ', exc_info=1)
else:
# deleteVM(vm_name, V1DeleteOptions())
ignore_pushing = False
step1_done = False
try:
jsondict = client.CustomObjectsApi(
).get_namespaced_custom_object(group=GROUP,
version=VERSION,
namespace='default',
plural=PLURAL,
name=vm_name)
except ApiException, e:
if e.reason == 'Not Found':
logger.debug(
'**VM %s already deleted, ignore this 404 error.' %
vm_name)
ignore_pushing = True
except Exception, e:
logger.error('Oops! ', exc_info=1)
def _retrieve_group(self, sync_source='MDM', **kwargs):
rs = self._process_once('retrieve', 'Groups', sync_source)
if not kwargs:
logger.warning(
'the retrieve condition is empty or none, it will return all groups in metanate'
)
return rs
# get a key and value from kwargs
key, value = self._get_key_value_from_dict(kwargs)
if not key:
logger.error(
'parameter did not contain "dn", "cn" or "objectguid"')
raise MDMiInvalidParameterError(
601, 'Invalid Parameter',
'parameter did not contain "dn", "cn" or "objectguid"')
for r in rs.content:
if isinstance(r, tuple) and len(r) == 2 and isinstance(r[1], dict):
if key == 'dn':
if r[0].lower() == value.lower():
return r
else:
r1 = self._convert_dict_key_to_lower(r[1])
if r1.has_key(key):
if isinstance(r1[key], list):
if key == 'objectguid':
if value in r1[key]:
return r
else: # cn is case insensitive
r2 = [
i.lower()
if isinstance(i, basestring) else i
for i in r1[key]
]
if value.lower() in r2:
return r
elif isinstance(r1[key], basestring):
if key == 'objectguid':
if r1[key] == value:
return r
else: # cn is case insensitive
if r1[key].lower() == value.lower():
return r
logger.error('cannot find group with condition: %s', kwargs)
return None
def _install_operation(self, operation):
# TODO: if operation specifies update directory, change to that
update_dir = settings.UpdatesDirectory
failed_to_download = False
urn_response = RvUrn.get_operation_urn(operation.type)
try:
self._download_updates(operation)
except Exception as e:
logger.error("Error occured while downloading updates.")
logger.exception(e)
failed_to_download = True
if not operation.install_data_list or failed_to_download:
error = RvError.UpdatesNotFound
if failed_to_download:
error = 'Failed to download packages.'
rvsof_result = RvSofResult(
operation.id,
operation.type,
'', # app id
[], # apps_to_delete
[], # apps_to_add
'false', # success
'false', # restart
error, # error
CreateApplication.null_application().to_dict(), # app json
urn_response,
RequestMethod.PUT)
self._send_results(rvsof_result)
else:
if operation.type == RvOperationValue.InstallAgentUpdate:
self._agent_update(operation, update_dir)
# TODO(urgent): remove this, only for testing
#elif operation.type == RvOperationValue.InstallCustomApps:
# self._agent_update(operation, update_dir)
else:
self._regular_update(operation, update_dir)
def get_all_agents_data(platform_access: PlatformAccess) -> Optional[List[Dict]]:
"""
Returns AgentData for all agents
Reference: https://github.com/own-dev/own-agent-open/blob/master/docs/APIDescription.md#get-agentdata
:return:
[
{
"id": int,
"description": str,
"salary": float,
"capacity": int,
"status": str,
"agentsUser": {
"id": int,
"firstName": str,
"lastName": str,
"email": str
},
"_links": [
{
"rel": str,
"href": str
},
{
"rel": str,
"href": str
}
]
}, ..
]
"""
try:
http_method = 'GET'
detail = 'agentData'
postfix = 'agentdata'
response = make_request(platform_access=platform_access,
http_method=http_method,
url_postfix=postfix,
detail=detail)
response.raise_for_status()
except Exception as excpt:
logger.error(OWN_ADAPTER_NAME, f'Error while updating database: {excpt}', response)
return None
else:
logger.debug(OWN_ADAPTER_NAME, f'Successfully updated the database: {response}', response)
if response:
return response.json().get('agentsData', None)
return []
def download(data: SubscribeCrawl):
try:
proxies = None
timeout = 10
if isinstance(data.rule, dict):
rule = VariableManager(data.rule)
if rule.get_conf_bool("need_proxy", default=False):
proxies = global_variable.get_conf_dict(
"PROXIES_CRAWLER",
default={
"http": "socks5://127.0.0.1:10808",
"https": "socks5://127.0.0.1:10808",
},
)
if data.rule.get("timeout"):
try:
timeout = int(data.rule.get("timeout"))
except:
timeout = 10
try:
headers = {
"User-Agent": global_variable.get_user_agent(),
"Connection": "close",
}
rsp = requests.get(data.crawl_url,
headers=headers,
timeout=timeout,
proxies=proxies)
if rsp.status_code == 200:
return rsp.content.decode("utf-8")
except (
requests.exceptions.RequestException,
requests.exceptions.RequestsWarning,
requests.exceptions.Timeout,
requests.exceptions.ConnectionError,
):
pass
except:
logger.error("err: {}".format(traceback.format_exc()))
except:
logger.error("err: {}".format(traceback.format_exc()))
return None
def fit_predict_thres(cls,
X_train,
Y_train,
X_test,
Y_test,
cnum,
label_list,
ignore_thres=None,
decision_thres=None):
"""
Any likelihood lower than a threshold is not counted as classification score
Params
======
ignore_thres = if not None or larger than 0, likelihood values lower than
ignore_thres will be ignored while computing confusion matrix.
"""
cls.fit(X_train, Y_train)
assert ignore_thres is None or ignore_thres >= 0
if ignore_thres is None or ignore_thres == 0:
Y_pred = cls.predict(X_test)
score = skmetrics.accuracy_score(Y_test, Y_pred)
cm = skmetrics.confusion_matrix(Y_test, Y_pred, label_list)
else:
if decision_thres is not None:
logger.error(
'decision threshold and ignore_thres cannot be set at the same time.'
)
raise ValueError
Y_pred = cls.predict_proba(X_test)
Y_pred_labels = np.argmax(Y_pred, axis=1)
Y_pred_maxes = np.array([x[i] for i, x in zip(Y_pred_labels, Y_pred)])
Y_index_overthres = np.where(Y_pred_maxes >= ignore_thres)[0]
Y_index_underthres = np.where(Y_pred_maxes < ignore_thres)[0]
Y_pred_overthres = np.array(
[cls.classes_[x] for x in Y_pred_labels[Y_index_overthres]])
Y_pred_underthres = np.array(
[cls.classes_[x] for x in Y_pred_labels[Y_index_underthres]])
Y_pred_underthres_count = np.array(
[np.count_nonzero(Y_pred_underthres == c) for c in label_list])
Y_test_overthres = Y_test[Y_index_overthres]
score = skmetrics.accuracy_score(Y_test_overthres, Y_pred_overthres)
cm = skmetrics.confusion_matrix(Y_test_overthres, Y_pred_overthres,
label_list)
cm = np.concatenate((cm, Y_pred_underthres_count[:, np.newaxis]),
axis=1)
return score, cm
def do_parse(self, datastr):
'''
Parses data string into base task struct dict.
Params:
datastr: Task struct string.
Return:
LIST of the task struct dict.
Number of the tasks
Exception message.
'''
try:
jsonobj = json.loads(datastr, encoding="utf-8")
if (isinstance(jsonobj, dict)): #error message is a dict
if (jsonobj.has_key('ErrorCode')):
logger.error(
'No valid task found, error message from response: %s'
% datastr)
return [], 0
#tasks in a List
self.m_tasks_num = len(jsonobj)
idx = 0
# format base task structure
while idx < self.m_tasks_num:
if not jsonobj[idx].has_key('name'):
# Not as task structure
self.m_tasks_num = idx
break
item = {
'owner': jsonobj[idx]['owner'],
'priority': jsonobj[idx].get('priority'),
'enqueue_timestamp': jsonobj[idx]['enqueue_timestamp'],
'account': int(jsonobj[idx]['account']),
'name': int(jsonobj[idx]['name']),
'tags': jsonobj[idx].get('tags'),
'task_retries': int(jsonobj[idx]['task_retries']),
'payload_base64': jsonobj[idx].get('payload_base64'),
}
self.m_tasks_list.append(item)
idx = idx + 1
return self.m_tasks_list, self.m_tasks_num
except Exception, e:
logger.error(e)
raise Exception('Invalid input! %s' % e)
def get_available_updates(self):
"""Get application instances of the packages in need of update.
"""
logger.info('Getting available updates.')
self._apt_update_index()
#installed_packages = self._get_installed_packages()
update_packages = self.check_available_updates()
update_pkgs_data = self.get_available_updates_data(update_packages)
apps = []
# For logging
i = 1
amount_of_packages = len(update_packages)
for package in update_packages:
pkg_name = package[0]
try:
app = self._create_app_from_dict(update_pkgs_data[pkg_name],
True)
# Slow, but as accurate as apt-get,
# returns a list of dictionaries which includes all deps
# [{'name': .. , 'version' : .. , 'app_id' : ..}]
app.dependencies = self._get_app_dependencies(app.name)
apps.append(app)
logger.debug(
'Done getting info for {0} out of {1}. Package name: {2}'.
format(i, amount_of_packages, app.name))
i += 1
except Exception as e:
logger.error('get_available_updates failed for: ' + pkg_name)
logger.exception(e)
# Append all dependencies' file_data to app's file_data.
# Must be done after getting all apps.
for app in apps:
self._append_file_data(app, apps)
return apps
def main():
# 将一些提交更新为wait，模拟实时比赛
# init_db = MySQLdb.connect(DATABASE_INFO['ip'], DATABASE_INFO['user'],
# DATABASE_INFO['pwd'], DATABASE_INFO['database'], charset='utf8')
# result2wait(init_db)
# 正式过程
try:
que = Queue.Queue()
db = MySQLdb.connect(DATABASE_INFO['ip'], DATABASE_INFO['user'],
DATABASE_INFO['pwd'], DATABASE_INFO['database'], charset='utf8')
except Exception as e:
logger.error(e)
raw_input('please check database information in utils.py, press enter to exit.')
return
try:
# 根据json文件里的用户密码数量创建对应个数worker
info = json.load(open(user_json_file))
for ele in info:
uid = ele['user']
pwd = ele['pwd']
worker = SubmitWorker(uid, pwd, que)
worker.daemon = True
worker.start()
logger.info('start {num} submit worker.'.format(num=len(info)))
except Exception as e:
logger.error(e)
raw_input('please check user-pwd.json file, press enter to exit.')
return
# problem_id 1039 solution_id 5054
try:
logger.info('start monitor.')
flag = True
while True:
if flag:
logger.info('try getting queueing submission.')
flag = wait2que(db, que)
que.join()
if flag:
logger.info('update queueing submission end.')
time.sleep(3)
except KeyboardInterrupt:
print ''
logger.warning('user abort.')
def consulta_empresa(update, context):
logger.info('empresa')
try:
# fim = datetime.now()
# inicio = fim - timedelta(days=90)
cnpj = update.message.text
user_name = update.message.from_user.username
logger.info('%s consultando empresa %s' % (user_name, cnpj))
r = requests.get(APIURL + 'consulta_empresa_text/%s' % cnpj,
verify=False)
update.message.reply_text(r.text)
except Exception as err:
text = str(err)
logger.error(err, exc_info=True)
update.message.reply_text(text)
return start(update, context)
def get_files_count_per_batch(intensors_desc, fileslist):
# get filesperbatch
filesize = get_files_datasize(fileslist[0][0])
tensorsize = intensors_desc[0].realsize
if filesize == 0 or tensorsize % filesize != 0:
logger.error('arg0 tensorsize: {} filesize: {} not match'.format(
tensorsize, filesize))
raise RuntimeError()
files_count_per_batch = (int)(tensorsize / filesize)
#runcount = math.ceil(len(fileslist) / files_count_per_batch)
runcount = len(fileslist[0]) // files_count_per_batch
logger.info(
"get filesperbatch files0 size:{} tensor0size:{} filesperbatch:{} runcount:{}"
.format(filesize, tensorsize, files_count_per_batch, runcount))
return files_count_per_batch, runcount
class AirwatchRequest:
'''
class AirwatchRequest
Attributes:
None
Notes:
execute() take requests sent from AW and delegate to airwatch_handler
'''
def __init__(self, version, type):
if version != "1":
raise web.NotFound("Not support Version %s" % version)
self.type = type
self.version = version
def execute(self):
try:
dict = {"data": web.data(), "headers": {"Content-Type": web.ctx.env.get('CONTENT_TYPE')}, "type": self.type}
AirwatchHandler(dict).handle()
monitorutils.count_task("notificationrestservice", True)
return web.OK()
except AuthenticationFailureException, e:
logger.error('Notification REST Service - %s' % e)
monitorutils.count_task("notificationrestservice", False)
raise web.Unauthorized()
except ValueError, e:
logger.error('Notification REST Service - %s' % e)
monitorutils.count_task("notificationrestservice", False)
raise web.NotAcceptable()
def __dispatch_get_fee_service(self, **kwargs):
"""
处理获取费用请求
Keyword Args:
room_id: 要关闭的从机的房间号
"""
if not self.__started:
logger.error('主控机未启动')
raise RuntimeError('主控机未启动')
get_fee_service = GetFeeService.instance()
room_id = kwargs.get('room_id')
if room_id is None:
logger.error('缺少参数room_id')
raise RuntimeError('缺少参数room_id')
return get_fee_service.get_current_fee(room_id)
def read_json_file(filename):
"""
Read data from json file.
Args:
filename: file name with "json" extension
Returns:
dict: content of the file.
Raises:
FileNotFoundError: if file doesn't exist.
"""
try:
with open(filename) as f:
return json.load(f)
except FileNotFoundError:
logger.error("File not found. Check file name and continue\n")
def send_keys_one_by_one(self, element, keys, clear=True):
"""
Send keys in an element letter by letter. This is to cover the issues that sometimes the normal send_keys
does not work properly
"""
try:
element = self.find_element(element)
if clear:
element.clear()
for letter in keys:
element.send_keys(letter)
if self.browser_name == 'firefox': # sometimes still fails on firefox
self.pause(0.1)
except Exception as ex:
logger.error(ex)
raise Exception(ex)
def valid(self) -> bool:
logger.debug("[Model] validate")
if not self._validate_json(self.collection):
logger.error("[Model] data is invalid.")
self.errors.append(
Exception(
"Invalid json format. Please pass in an array of json that keeps `title`, `url` and `abstract` as keys."
))
return False
elif not self._validate_blank(self.collection):
logger.error("[Model] data is invalid.")
self.errors.append(Exception("The result passed was empty."))
return False
else:
logger.debug("[Model] data is valid")
return True
def dump_graph(self):
tasks = []
[
tasks.append(x.to_dict()) for x in self.task_idx.values()
if x.id != "Sink"
]
with open(config.json_dump_path, 'w') as dump_path:
json.dump({"tasks": tasks}, dump_path, indent=4)
# check failed task
err = 0
for t in tasks:
if t.get("status") == "FINISH_FAILED":
logger.error("===========> task failed: %s" % str(t))
err += 1
return err
def do_logout(self, input_string):
self.setup_argparse(usage="$ logout",
description='Logout current logged in user')
# validate arguments
try:
args = self.parser.parse_args(input_string)
except ValueError:
return
# logging out user
response = requests.post(config.CS_LOGOUT_URL, json={})
response = response.json()
if response.get("error"):
logger.error("Error: {}".format(response.get("error")))
return
logger.info("Cya :)")
def except_param_adptor(*args, **kwargs):
try:
return fun(*args, **kwargs)
except:
exec_info = sys.exc_info()
track_back_data = "%s Error!!!!!!!!!!!!!\n" % fun.__name__
track_back_data += "args:%s" % str(args) + '\n'
track_back_data += "kwargs:%s" % str(kwargs) + '\n'
track_back_data += str(exec_info[0]) + ":" + str(
exec_info[1]) + '\n'
track_back_data += traceback.format_exc()
logger.error(track_back_data)
if is_raise:
raise
if default_result:
return default_result
def limit_assert(self, action, quantity):
"""
资产限制
:param action:
:param quantity:
:return:
"""
asserts = copy.copy(self.assets)
if not asserts.assets:
logger.error("asserts not init", caller=self)
return False
asset = asserts.assets.get(self.symbol)
if not asset:
logger.error(self.symbol, "no asset", caller=self)
return False
return True
async def delete_location(sid: str, location_id: int):
pr: PlayerRoom = game_state.get(sid)
if pr.role != Role.DM:
logger.warning(f"{pr.player.name} attempted to rename a location.")
return
location = Location.get_by_id(location_id)
if location.players.count() > 0:
logger.error(
"A location was attempted to be removed that still has players! This has been prevented"
)
return
location.delete_instance(recursive=True)
def run_command(*args):
output = ""
try:
process = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines = True)
return_code = process.wait()
output = process.stdout.read()
logger.debug("%s : code %s \n stdout=\n %s \n\n", process.args,
return_code, output)
except Exception as e:
logger.error("Error %s \nfor command %s", e, " ".join(args))
finally:
return escape_output(output.strip())
def save_repo_state_filesystem(self):
repodir = os.path.join(self.path, 'x86_64')
pkgs = set(p for p in os.listdir(repodir)
if '.pkg.' in p and not p.endswith('.sig'))
for pkg in pkgs:
pkg = os.path.basename(pkg)
try:
pkg, version, rel, suffix = pkg.rsplit('-', 3)
except ValueError:
logger.error("unexpected pkg: " + pkg)
continue
self.pkgs_fs.add('{0}|{1}-{2}'.format(pkg, version, rel))
self.pkg_count_fs = len(self.pkgs_fs)
def add_group(self, dn, object_guid=None, **kwargs):
"""add an group in metanate
Parameters:
dn: the ldap dn of group
object_guid: the id of the group
"""
if not dn:
logger.error('the format of group dn: %s is not valid', dn)
raise MDMiInvalidParameterError(
601, 'Invalid Parameter',
'the format of group dn: %s is not valid' % dn)
if not object_guid:
object_guid = self.generate_guid()
return self.replace_group(dn, object_guid, 'add', **kwargs)
