def on_recieve(self, msg, src, dst, paras='', paras_to_save='', index=0): """ The start of the action to a message recieved. This keyword starts a sub-procedure used as the reaction to a message recieved. The sub-procedure ended when next on_recieve met. The action set to certain message is only valid for one test case. """ if self.msg_buff is None: self._init_msg_buff() ne_link = self._topology.link(src, dst) if ne_link is None: logger.warn('No link between NEs to send message. ' '%s<-%s: %s' % (dst, src, msg)) return if not (ne_link.simulator == dst and ne_link.dut == src): logger.warn('Not a message from dut to simulator. ' '%s<-%s: %s' % (dst, src, msg)) return msg_str = self._compose_on_recv(msg, paras, paras_to_save, index) msg_buff = self._get_msg_buff(ne_link.names[0]) msg_buff.append(msg_str) logger.info('"send %s" added to buffer of %s' % (msg, ne_link.names[0]))
def check(self, expr, info=''): if self.msg_buff is None: self._init_msg_buff() self.chk_lst.check(expr, info) logger.info('"Check" added to check list')
def send(self, msg, src, dst, paras='', paras_to_save='', delay=0): """ Composing a message sending from simulator to dut. """ if self.msg_buff is None: self._init_msg_buff() ne_link = self._topology.link(src, dst) if ne_link is None: logger.warn('No link between NEs to send message. ' '%s->%s: %s' % (src, dst, msg)) return if not (ne_link.simulator == src and ne_link.dut == dst): logger.warn('Not a message from simulator to dut. ' '%s->%s: %s' % (src, dst, msg)) return msg_str = self._compose_send(msg, paras, paras_to_save, delay) msg_buff = self._get_msg_buff(ne_link.names[0]) msg_buff.append(msg_str) logger.info('"send %s" added to buffer of %s' % (msg, ne_link.names[0])) if self.trigger is None and not self._on_recv_found: self.trigger = msg_buff.ne_name logger.info('Found trigger: %s' % src)
def _dpkg_install(self, update_dir=None): """Install an update or install a new package. If an update directory is given, it must be made sure that the directory contains all of the dependencies for ALL of the packages inside the directory. """ if update_dir: install_command = ['dpkg', '-i', '-R', update_dir] try: result, err = self.utilcmds.run_command(install_command) if err: raise Exception(err) logger.info( ('debhandler.py/_dpkg_install: ' 'Installed all packages in: ') + update_dir) return 'true', '' except Exception as e: logger.error( ('debhandler.py/_dpkg_install: ' 'Failed to install packages in: ') + update_dir) return 'false', str(e) else: logger.info( 'debhandler.py/_dpkg_install: No directory provided.') return 'false', 'Update dir: ' + update_dir
def run(self): logger.info('Devices\' information sync with airwatch started at %s under account %d!' % (str(time.time()), self.aw_account)) try: logger.info('Step 1: Get device information from RS.') rs_retval = self._get_devinfo_from_rs() logger.debug('Account %d includes %d device needed sync with airwatch!' % (self.aw_account, rs_retval)) if rs_retval <= 0: # report return # exit, no work to do logger.info('Step 2: Get device information from airwatch.') aw_retval = self._get_devinfo_from_aw(self.devinfo_original_queue, self.devinfo_refresh_queue) logger.debug('Account %d, get %d device information from airwatch!' % (self.aw_account, aw_retval)) if (aw_retval != rs_retval): #The devices do not exist in airwatch needed to be updated as unenroll status. logger.warning('Account %d, Original device number (%d) and refresh device number (%d) NOT match!' % (self.aw_account, rs_retval, aw_retval)) logger.info('Step 3: Sync device information to RS.') sync_retval = self._sync_devinfo() logger.debug('Account %d, sync %d device information with RS finished!' % (self.aw_account, sync_retval)) if (aw_retval != sync_retval): #causes of updating RS fail logger.warning('Account %d, Refresh device number (%d) and Sync device number (%d) NOT match!' % (self.aw_account, aw_retval, sync_retval)) # Step 4: Update the defect devices to be unenroll. if self.m_defect_devs: defect_cnt = len(self.m_defect_devs) logger.info('Step 4: Set %d devices to be "unenroll" status!' % defect_cnt) ok_cnt = self._unenroll_dev_status(self.m_defect_devs) if defect_cnt != ok_cnt: logger.warning('Account %d, Set %d devices to be "unenroll status failed!"' % (self.aw_account, (defect_cnt - ok_cnt))) # Step 5: Report except Exception, e: logger.error(repr(e))
def _compute_detailed_scores(self, item_ids, query_item_ids=None, max_terms=20): # if set to None we assume previously queried items if query_item_ids is None: query_item_ids = self.item_ids # if the query vector is different than previously computed # or not computed at all, we need to recompute it. if not hasattr(self, 'q') or query_item_ids != self.item_ids: if not self.is_valid_query(query_item_ids): return [] else: logger.info('Computing the query vector ...') self._make_query_vector() # computing the score for each item scores = [] for id in item_ids: if id not in self.item_id_to_index: scores.append(utils._O(total_score=0, scores=[])) continue xi = self.X[self.item_id_to_index[id]] xi_ind = xi.indices feat = (self.index_to_feat[i] for i in xi_ind) qi = self.q.transpose()[xi_ind] qi = scipy.asarray(qi).flatten() sc = sorted(zip(feat, qi), key=lambda x: (x[1], x[0]), reverse=True) total_score = qi.sum() scores.append(utils._O(total_score=total_score, scores=sc[0:max_terms])) return scores
def _reset_notify(self): from service_mgr.lib.service.service_main import ServiceMgr services = ServiceMgr().get_run_services(US_DEVICE_END) [device_type_notify(service.control_rpc, self.get_device_type()) for service in services] logger.info("DeviceTypeMgr _reset_notify services:%s, device_type:%s" % ([service.id for service in services], self.get_device_type()))
def top_phrases(self, msg): logger.info("%s: sending top phrases to user %s" % (msg.text, user_log_string(msg))) lines = ["Top phrases (use '/say ID' to choose phrase):"] for audio_id, count in self.dataset.top_phrases.most_common(5): lines.append( "*[%2d]* _%s_ (%d reproductions)" % (audio_id, self.dataset.get_audio_data(audio_id)['title'], count)) self.reply_to(msg, "\n".join(lines), parse_mode="Markdown")
def send_phrase(self, msg, audio_id): if 0 <= audio_id < len(self.dataset.data_keys): audio, duration = self.dataset.get_phrase(audio_id) logger.info("%s: sending phrase #%d to user %s" % (msg.text, audio_id, user_log_string(msg))) self.send_audio(msg.chat.id, audio, duration) else: bot.send_random_phrase(msg)
def main(): rel_info = ReleaseInfo() start_directory = os.getcwd() parse_args(rel_info) try: setup_test_branch(rel_info) merge_branches(rel_info) test_tag_doc(rel_info) if rel_info.args.merge_only: return build_platform(rel_info) except KeyboardInterrupt: pass except: logger.exception("[FAILED] Please check your changes. "\ "You can not pass this checker unless your branch "\ "can be merged without conflicts.") logger.info("Note: all repos are in test branch %s" % rel_info.test_branch) else: logger.info( "[PASS] Your changes can be successfully merged and build.") finally: cleanup(rel_info) os.chdir(start_directory)
def do_access(self, resource, method_name='POST', data=None, headers=None): """request resource in specified HTTP method Parameters: resource: an absolute path which supplied by server. method_name: HTTP METHOD, can be 'GET', 'POST', 'PUT' or 'DELETE', the default is 'POST' data: the data should transfer to server. headers: HTTP headers, should in dict form. """ boundary = templates.generate_boundary() http_headers = self._generate_headers(boundary) if headers: http_headers.update(headers) if data: body = self._generate_post_body(data, boundary) else: body = "" http_headers.pop('Content-Type') logger.info('metanate request body: %s', body) r = super(self.__class__, self).do_access(resource, method_name='POST', data=body, headers=http_headers) if r.code == 200: if not self.session_id: self.session_id = r.session_id logger.info('HTTP response header X-Schemus-Session is: %s', self.session_id) if r.content_type == "text/ldif": r.content = templates.parse_ldif(r.content) elif r.content == "application/json": r.content = json.loads(r.content) else: logger.error('metanate request: %s error occurred: %s: %s', resource, r.code, r.content) r.content = {'ErrorCode': r.code, 'Message': r.reason, 'Detail': r.content} return r
def process_result(self, result, live_server, token): """ Munge the result from a client and add the data into our repository. """ # Record the data from our transaction in the Tokens # Ensures that duplicated work is not accepted after the # token has been consumed (to prevent previous generation's # evals from affecting the current gen) if not result.has_key('FAILED'): if token in self.Tokens.keys(): self.data_lock.acquire() if token[0] == 'map': (shard, shuffle_keys) = result.values()[0] self.shuffle_keys.update(shuffle_keys) logger.info('Have %d shuffle keys (only alpha)' % (len(self.shuffle_keys))) self.map_result_shards.append(shard) elif token[0] == 'shuffle': self.shuffle_result_shards.append(result.values()[0]) self.data_lock.release() # Consume the token if we've performed enough Evals self.print_complete(token, self.Tokens, live_server, live_servers, idle_servers) del self.Tokens[token] else: # Delete extraneous data if token[0] == 'map': (shard, shuffle_keys) = result.values()[0] os.remove(shard)
def _untar_files(self, directory): """ Scans a directory for any tar files and 'untars' them. Scans recursively just in case there's tars within tars. Deletes tar files when done. @param directory: Directory to be scanned. @return: Nothing """ tars = glob.glob(os.path.join(directory, '*.tar*')) if not tars: return import tarfile try: for tar_file in tars: tar = tarfile.open(tar_file) tar.extractall(path=directory) tar.close() os.remove(tar_file) self._untar_files(directory) except OSError as e: logger.info("Could not extract tarball.") logger.exception(e)
def start_net(self, name): logger.info("Start network " + name) if name in self.networks: conn = libvirt.open(self.urls[self.networks[name].htype]) else: conn = libvirt.open(self.def_connection) try: net = conn.networkLookupByName(name) if not net.isActive(): logger.debug("Network registered in libvirt - start it") net.create() else: logger.debug("Network already active") except libvirt.libvirtError: try: logger.debug("No such network in libvirt") net = self.networks[name] except KeyError: msg = "Can't found network {0!r}".format(name) logger.error(msg) raise CloudError(msg) xml = xmlbuilder.XMLBuilder('network') xml.name(name) xml.bridge(name=net.bridge) with xml.ip(address=net.ip, netmask=net.netmask): xml.dhcp.range(start=net.ip1, end=net.ip2) logger.debug("Create network") conn.networkCreateXML(str(xml))
def finish(self, link): """ The stop point of test case. At least one stop point must be exist in each test case. This keyword represents the end of the test case. Test tool will send stop signal to dispatcher then the dispatcher will notify all other tools to stop. After all tools stopped, the dispatcher will send test reports to the verification module to be verified. """ if self.msg_buff is None: self._init_msg_buff() src, dst = link.split('--') ne_link = self._topology.link(src, dst) if ne_link is None: logger.warn('No link between NEs to send message. ' '%s--%s: Stop' % (src, dst)) return if ((not (ne_link.simulator == src and ne_link.dut == dst)) and (not (ne_link.simulator == dst and ne_link.dut == src))): logger.warn('Finish on unassigned link. ' '%s--%s: Stop' % (src, dst)) return msg_str = self._compose_finish() msg_buff = self._get_msg_buff(ne_link.names[0]) msg_buff.append(msg_str) logger.info('"Finish" added to buffer of %s' % ne_link.names[0])
def user_insert(self, user_name, password, jid, jid_pwd, type, device_type="", des=""): """ 用户插入 :param user_name: 用户名 :param password: 用户密码 :param jid: 用户jid :param jid_pwd: 用户jid密码 :param type: 用户类型 :param device_type: 设备类型 :param des: 描述 :return: {"result"} """ if not user_name\ or not password\ or not jid\ or not jid_pwd\ or type is None: return {'result': error_code.ERROR_PARAMS_ERROR} user_info = {"user_name": user_name, "password": password, "des": des, "jid": jid, "jid_pwd": jid_pwd, "device_type": device_type, "type": type, "create_time": datetime.datetime.now()} g_RedisCoordinator.set(user_name, user_info) logger.info("RpcHandler::user_insert, user_name:%s, password:%s, jid:%s, jid_pwd:%s, device_type:%s, type:%s, des:%s" % (user_name, password, jid, jid_pwd, device_type, type, des)) return {'result': error_code.ERROR_SUCCESS}
def __create_distro(self, compose, distro_file): rhel_version = self.get_rhel_version(compose) if rhel_version == 5: compose_url = "http://download.englab.nay.redhat.com/pub/rhel/rel-eng/%s/tree-x86_64/" % compose elif rhel_version == 6: # compose_url = "http://download.englab.nay.redhat.com/pub/rhel/rel-eng/%s/%s/Server/x86_64/os/" % (compose, self.get_rhel_version(compose)) compose_url = "http://download.englab.nay.redhat.com/pub/rhel/rel-eng/%s/compose/Server/x86_64/os/" % compose elif rhel_version == 7: compose_url = "http://download.englab.nay.redhat.com/pub/rhel/rel-eng/%s/compose/Server/x86_64/os/" % compose if not self.__check_file_exist(distro_file): cmd = ('cat <<EOF > %s\n' '[General]\n' 'arch : x86_64\n' 'breed : redhat\n' 'comment :\n' '\n' 'kernel : %simages/pxeboot/vmlinuz\n' 'initrd : %simages/pxeboot/initrd.img\n' 'kernel_options : biosdevname=0 reboot=pci\n' 'kernel_options_post :\n' 'ks_meta :\n' 'mgmt_classes :\n' '\n' 'os_version : rhel%s\n' 'redhat_management_key :\n' 'redhat_management_server :\n' 'template_files :\n' 'EOF' % (distro_file, compose_url, compose_url, rhel_version) ) logger.info("Created distro file: %s" % distro_file) self.run(cmd) else: logger.info("Distro file: %s already existed ..." % distro_file)
def init_platform(): config_file = cfg_from_file('config.yml') default_file = cfg_from_file('default.yml') logger.info(pprint.pformat(default_file)) logger.info(pprint.pformat(config_file)) merge_a_into_b(config_file, config) merge_a_into_b(default_file, default) default.best_model_path = '' if default.gpu == '': default.gpu = None if default.gpu is not None: os.environ["CUDA_VISIBLE_DEVICES"] = default.gpu default.distributed = default.world_size > 1 if default.distributed: dist.init_process_group(backend=default.dist_backend, init_method=default.dist_url, world_size=default.world_size) default.lr_epoch = [int(ep) for ep in default.lr_step.split(',')] if default.seed is not None: seed = default.seed np.random.seed(seed) random.seed(seed) torch.manual_seed(seed) cudnn.deterministic = True
def _create_indexes(self, ids, fts): logger.info("Creating indices ...") self.item_id_to_index = dict(ids) self.index_to_item_id = dict((i, id) for id, i in ids.iteritems()) self.index_to_feat = dict((i, ft) for ft, i in fts.iteritems()) self.no_items = len(ids) self.no_features = len(fts)
def wait(self, link, tm): """ Wait for certain seconds specified by 'tm'. 'tm' is the time to be waited in millisecond. """ if self.msg_buff is None: self._init_msg_buff() src, dst = link.split('--') ne_link = self._topology.link(src, dst) if ne_link is None: logger.warn('No link between NEs to send message. ' '%s->%s: Wait' % (src, dst)) return if ((not (ne_link.simulator == src and ne_link.dut == dst)) and (not (ne_link.simulator == dst and ne_link.dut == src))): logger.warn('Stop on unassigned link. ' '%s-%s: Stop' % (src, dst)) return msg_str = self._compose_wait(tm) msg_buff = self._get_msg_buff(ne_link.names[0]) msg_buff.append(msg_str) logger.info('"wait %s" added to buffer %s' % (tm, ne_link.names[0]))
def uninstall_application(self, uninstall_data): """ Uninstalls applications in the /Applications directory. """ success = 'false' error = 'Failed to uninstall application.' restart = 'false' #data = [] uninstallable_app_bundles = os.listdir('/Applications') app_bundle = uninstall_data.name + ".app" if app_bundle not in uninstallable_app_bundles: error = ("{0} is not an app bundle. Currently only app bundles are" " uninstallable.".format(uninstall_data.name)) else: uninstaller = Uninstaller() success, error = uninstaller.remove(uninstall_data.name) logger.info('Done attempting to uninstall app.') return UninstallResult(success, error, restart)
def retrieve_data(self, link, result, operation, paras='', start_message='', end_message=''): if self.msg_buff is None: self._init_msg_buff() src, dst = link.split('--') ne_link = self._topology.link(src, dst) if ne_link is None: logger.warn('No link between NEs to send message. ' '%s--%s: Retrieve %s' % (src, dst, operation)) return if ((not (ne_link.simulator == src and ne_link.dut == dst)) and (not (ne_link.simulator == dst and ne_link.dut == src))): logger.warn('Retrieve data on unassigned link. ' '%s--%s: Retrieve %s' % (src, dst, operation)) return msg_str = self._compose_retrieve_data(result, operation, paras, start_message, end_message) msg_buff = self._get_msg_buff(ne_link.names[0]) msg_buff.append(msg_str) logger.info('"Retrieve" added to buffer of %s' % ne_link.names[0])
def _apt_purge(self, package_name): purged = 0 purge_command = [self.APT_GET_EXE, 'purge', '-y', package_name] try: result, err = self.utilcmds.run_command(purge_command) if err: raise Exception(err) found = re.search('\\d+ to remove', result) if found: amount_purged = found.group().split(' ')[0] purged = int(amount_purged) if purged > 0: logger.debug( 'Successfuly removed {0} packages.'.format(purged) ) return 'true', '' else: logger.info('No packages were removed.') return 'false', 'No packages removed.' except Exception as e: logger.error('Problem while uninstalling package: ' + package_name) logger.exception(e) return 'false', str(e)
def run(self): while self.processor.shared.paused(): time.sleep(1) self.ircname = self.host + ' ' + self.getname() logger.info("joining IRC") while not self.processor.shared.stopped(): client = irc.client.IRC() try: c = client.server().connect('irc.freenode.net', 6667, self.nick, self.password, ircname=self.ircname) except irc.client.ServerConnectionError: logger.error('irc', exc_info=True) time.sleep(10) continue c.add_global_handler("welcome", self.on_connect) c.add_global_handler("join", self.on_join) c.add_global_handler("quit", self.on_quit) c.add_global_handler("kick", self.on_kick) c.add_global_handler("whoreply", self.on_who) c.add_global_handler("namreply", self.on_name) c.add_global_handler("disconnect", self.on_disconnect) c.set_keepalive(60) self.connection = c try: client.process_forever() except BaseException as e: logger.error('irc', exc_info=True) time.sleep(10) continue logger.info("quitting IRC")
def add_vpn_ondemandrules(self, node, vpn_bypass_except): if node is None: logger.info('OnDemandRules NOT found!'); return; attr_val = None; if vpn_bypass_except and isinstance(vpn_bypass_except, dict): attr_val = vpn_bypass_except.get('attributes'); attr_key = None; attr_key_val = None; if attr_val and isinstance(attr_val, dict): for (k, v) in attr_val.items(): if k == 'LocalNetworkAccessKey': if v: attr_key = v[0]; #get value from list if k == 'LocalNetworkAccessValue': if v: attr_key_val = v[0]; rules_array = node.getchildren(); #array in plist, list in python key_is_found = False; for element in rules_array: #element should be a dict for item in element: if item.text == attr_key: key_is_found = True; #found! continue; if key_is_found: self.add_vpnexception(item, attr_key_val); key_is_found = False; #clear! break;
def run(self): logger.info('dispatch thread is running...') while not self.__stop_event.isSet(): try: self.__event.wait(timeout=5) except Exception: pass else: if self.__cur_tasks: if self._run_tasks(): self.__event.set() else: self.__event.clear() else: self.__cur_tasks = g_task_dict.get() if not self.__cur_tasks: self.__task_handler_pool.kill_idle_handlers() self.__event.clear() continue if self._run_tasks(): self.__event.set() else: self.__event.clear() finally: pass logger.info('dispatch thread %s stopped', self.name)
def stop(self): logger.info('stopping dispatch thread %s...', self.name) self.__stop_event.set() #if not self.__busy.isSet(): self.__event.set() self.__task_handler_pool.kill_all_handlers()
def main(): """ The function gets all the user callback objects whose notify times are between the current time and 1 minute from now. For each callback object it calls the function notification_callback, each call is spawned as separate thread. This way, all users get concurrent notifications and if there are a large number of notifications we won't be taking up more than 1 minute to complete the execution. All the spawned threads are joined with this function, so that it waits until they complete their execution. @params: None @output: None """ now = datetime.datetime.now() now_time = datetime.time(now.hour, now.minute, now.second) xmins_frm_now = now + datetime.timedelta(seconds = 60) xmins_frm_now = datetime.time(xmins_frm_now.hour, xmins_frm_now.minute, xmins_frm_now.second) today = datetime.date.today() days = {0:'mon', 1:'tue', 2:'wed', 3:'thu', 4:'fri', 5:'sat', 6:'sun'} today = days[today.weekday()] user_callbacks = models.UserCallback.objects.filter(notify_time__gte=now_time, notify_time__lt=xmins_frm_now) logger.info("number of user callbacks to process: %d" % user_callbacks.count()) threads = [] for user_callback in user_callbacks: thread = threading.Thread(target = notification_callback, args = (user_callback, today,)) threads.append(thread) try: thread.start() except Exception, e: logger.error("error while running user callback: %s" % e)
def wechat_recv_param_wapper(self, *args, **kwargs): if not signature_checker(token, kwargs['timestamp'], kwargs['nonce'], kwargs['signature']): logger.error("wechat_recv_wapper, failed, access_token:%s args:%s, kwargs:%s" % (token, args, kwargs)) return "-1" # 如果没有内容尽是验证,则即可返回 if not self.request.body: return kwargs.get("echostr", "0") # 内容解析 crypt = WXBizMsgCrypt(token, encodingAESKey, appid) ret, xml_body = crypt.DecryptMsg(self.request.body, kwargs['msg_signature'], kwargs['timestamp'], kwargs['nonce']) assert ret == WXBizMsgCrypt_OK order_dic_body = xmltodict.parse(xml_body) dic_body = dict(order_dic_body["xml"]) kwargs['body'] = dic_body logger.info("%s:wechat_recv_wapper args:%s kwargs:%s" % (fun.__name__, args, kwargs)) fun(self, *args, **kwargs) return kwargs.get("echostr", "0")
def parse_result(self, res): retcode = res.status result = RestResult(retcode, res.reason) content_type = res.getheader('Content-Type') if content_type and content_type.find(";"): types = content_type.split(";") for t in types: t = t.strip() if t.startswith('charset'): result.charset = t else: result.content_type = t else: result.content_type = content_type content_length = res.getheader('Content_Length') if content_length: result.content_length = int(content_length) result.content = res.read() while len(result.content) < result.content_length: result.content += res.read() else: result.content = res.read() result.content_length = len(result.content) last_modified = res.getheader('Last-Modified') if last_modified: logger.info('HTTP response from %s, Last-Modified header is: %s', self.host, last_modified) result.last_modified = last_modified session_id = res.getheader('X-Schemus-Session') if session_id: logger.info('HTTP response from %s, X-Schemus-Session header is: %s', self.host, session_id) result.session_id = session_id return result
def cancel_private(update, context): message = update.message context.chat_data.clear() message.reply_text(context.bot_data.get("config").get("CANCEL_PRIVATE")) logger.info(f"Private: Cancel") return ConversationHandler.END
def get_state(self, evaluation=False): succeed = False error_cnt = 0 while (1): # Read value from shared memory try: memory_value = self.shrmem_r.read() except sysv_ipc.ExistentialError: print("No shared memory Now, python ends gracefully :)") logger.info("No shared memory Now, python ends gracefully :)") sys.exit(0) memory_value = memory_value.decode('unicode_escape') i = memory_value.find('\0') if i != -1: memory_value = memory_value[:i] readstate = np.fromstring(memory_value, dtype=float, sep=' ') try: rid = readstate[0] except: rid = self.prev_rid sleep(0.01) continue try: s0 = readstate[1:] except: print("s0 waring") sleep(0.01) continue if rid != self.prev_rid: succeed = True break else: wwwwww = "" error_cnt = error_cnt + 1 if error_cnt > 24000: error_cnt = 0 print( "After 3 min, We didn't get any state from the server. Actor " + str(self.config.task) + " is going down down down ...\n") sys.exit(0) sleep(0.01) error_cnt = 0 if succeed == False: raise ValueError('read Nothing new from shrmem for a long time') reward = 0 state = np.zeros(1) w = s0 if len(s0) == (self.params.dict['input_dim']): d = s0[0] thr = s0[1] samples = s0[2] delta_t = s0[3] target_ = s0[4] cwnd = s0[5] pacing_rate = s0[6] loss_rate = s0[7] srtt_ms = s0[8] snd_ssthresh = s0[9] packets_out = s0[10] retrans_out = s0[11] max_packets_out = s0[12] mss = s0[13] min_rtt = s0[14] self.local_counter += 1 if self.use_normalizer == True: if evaluation != True: self.normalizer.observe(s0) s0 = self.normalizer.normalize(s0) min_ = self.normalizer.stats() else: min_ = s0 - s0 d_n = s0[0] - min_[0] thr_n = s0[1] thr_n_min = s0[1] - min_[1] samples_n = s0[2] samples_n_min = s0[2] - min_[2] delta_t_n = s0[3] delta_t_n_min = s0[3] - min_[3] cwnd_n_min = s0[5] - min_[5] pacing_rate_n_min = s0[6] - min_[6] loss_rate_n_min = s0[7] - min_[7] srtt_ms_min = s0[8] - min_[8] snd_ssthresh_min = s0[9] - min_[9] packets_out_min = s0[10] - min_[10] retrans_out_min = s0[11] - min_[11] max_packets_out_min = s0[12] - min_[12] mss_min = mss - min_[13] min_rtt_min = min_rtt - min_[14] if self.use_normalizer == False: thr_n = thr_n thr_n_min = thr_n_min samples_n_min = samples_n_min cwnd_n_min = cwnd_n_min loss_rate_n_min = loss_rate_n_min d_n = d_n if self.max_bw < thr_n_min: self.max_bw = thr_n_min if self.max_cwnd < cwnd_n_min: self.max_cwnd = cwnd_n_min if self.max_smp < samples_n_min: self.max_smp = samples_n_min if self.min_del > d_n: self.min_del = d_n ################# Transfer all of the vars. to Rate/Max(Rate) space #cwnd_bytes= cwnd_n_min*mss_min #cwnd_n_min=(cwnd_bytes*1000)/srtt_ms_min #snd_ssthresh_min=(snd_ssthresh_min*mss_min*1000)/srtt_ms_min #packets_out_min=(packets_out_min*mss_min*1000)/srtt_ms_min #retrans_out_min=(retrans_out_min*mss_min*1000)/srtt_ms_min #max_packets_out_min=(max_packets_out_min*mss_min*1000)/srtt_ms_min #inflight_bytes=(packets_out-samples)*mss_min*1000 if min_rtt_min * ( self.params.dict['delay_margin_coef']) < srtt_ms_min: delay_metric = ( min_rtt_min * (self.params.dict['delay_margin_coef'])) / srtt_ms_min else: delay_metric = 1 reward = (thr_n_min - 5 * loss_rate_n_min) / self.max_bw * delay_metric if self.max_bw != 0: state[0] = thr_n_min / self.max_bw tmp = pacing_rate_n_min / self.max_bw if tmp > 10: tmp = 10 state = np.append(state, [tmp]) state = np.append(state, [5 * loss_rate_n_min / self.max_bw]) else: state[0] = 0 state = np.append(state, [0]) state = np.append(state, [0]) state = np.append(state, [samples / cwnd]) state = np.append(state, [delta_t_n]) state = np.append(state, [min_rtt_min / srtt_ms_min]) state = np.append(state, [delay_metric]) self.prev_rid = rid return state, d, reward, True else: return state, 0.0, reward, False
def hicup_alig_filt(self, params, genome_digest, genome_index, genome_loc, fastq1, fastq2, outdir_tar): """ This function aling the HiC read into a reference genome and filter them Parameters ---------- bowtie2_loc: genome_index: str location of genome indexed with bowtie2 digest_genome: str location of genome digested fastq1: str location of fastq2 file fastq2: str location of fastq2 Returns ------- Bool """ folder = os.path.split(outdir_tar)[0]+"/"+ \ os.path.split(outdir_tar)[1].split(".")[0] if os.path.isdir(folder) is False: os.mkdir(folder) index_files = { "1.bt2": genome_loc + ".1.bt2", "2.bt2": genome_loc + ".2.bt2", "3.bt2": genome_loc + ".3.bt2", "4.bt2": genome_loc + ".4.bt2", "rev.1.bt2": genome_loc + ".rev.1.bt2", "rev.2.bt2": genome_loc + ".rev.2.bt2" } logger.progress("Untar Index: " + genome_loc + ", " + genome_index) self.untar_index(genome_loc, genome_index, index_files["1.bt2"], index_files["2.bt2"], index_files["3.bt2"], index_files["4.bt2"], index_files["rev.1.bt2"], index_files["rev.2.bt2"]) hicup_args = [ "hicup", "--index", genome_loc, "--digest", genome_digest, fastq1, fastq2 ] hicup_args = hicup_args + params + [ "--bowtie2", "/home/compss/bin/bowtie2", "--outdir", folder ] logger.info("arguments for hicup:" + " ".join(hicup_args)) try: process = subprocess.Popen(" ".join(hicup_args), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) process.wait() logger.info("TARING output folder") tar_file = outdir_tar archive_name = os.path.split(outdir_tar)[1].split(".")[0] onlyfiles = [ f for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f)) ] tar = tarfile.open(tar_file, "w") for tmp_file in onlyfiles: tar.add(os.path.join(folder, tmp_file), arcname=os.path.join(archive_name, tmp_file)) os.remove(os.path.join(folder, tmp_file)) tar.close() shutil.rmtree(folder) shutil.move(tar_file, outdir_tar) for indexed_file in index_files: os.remove(index_files[indexed_file]) return True except IOError: return False
def add_task(self, task_obj): logger.info("putting to queue: %s - %s", task_obj.id, task_obj.cmd) self.task_queue.put(task_obj)
def _train(self): """ Main actor learner loop for advantage actor critic learning. """ logger.debug("Actor {} resuming at Step {}".format( self.actor_id, self.global_step.value())) bonuses = deque(maxlen=100) while (self.global_step.value() < self.max_global_steps): # Sync local learning net with shared mem s = self.emulator.get_initial_state() self.reset_hidden_state() self.local_episode += 1 episode_over = False total_episode_reward = 0.0 episode_start_step = self.local_step while not episode_over: self.sync_net_with_shared_memory(self.local_network, self.learning_vars) self.save_vars() rewards = list() states = list() actions = list() values = list() local_step_start = self.local_step self.set_local_lstm_state() while self.local_step - local_step_start < self.max_local_steps and not episode_over: # Choose next action and execute it a, readout_v_t, readout_pi_t = self.choose_next_action(s) new_s, reward, episode_over = self.emulator.next(a) total_episode_reward += reward # Update density model current_frame = new_s[..., -1] bonus = self.density_model.update(current_frame) bonuses.append(bonus) if self.is_master() and (self.local_step % 400 == 0): bonus_array = np.array(bonuses) logger.debug( 'π_a={:.4f} / V={:.4f} / Mean Bonus={:.4f} / Max Bonus={:.4f}' .format(readout_pi_t[a.argmax()], readout_v_t, bonus_array.mean(), bonus_array.max())) # Rescale or clip immediate reward reward = self.rescale_reward( self.rescale_reward(reward) + bonus) rewards.append(reward) states.append(s) actions.append(a) values.append(readout_v_t) s = new_s self.local_step += 1 global_step, _ = self.global_step.increment() if global_step % self.density_model_update_steps == 0: self.write_density_model() if self.density_model_update_flags.updated[ self.actor_id] == 1: self.read_density_model() self.density_model_update_flags.updated[ self.actor_id] = 0 next_val = self.bootstrap_value(new_s, episode_over) advantages = self.compute_gae(rewards, values, next_val) targets = self.compute_targets(rewards, next_val) # Compute gradients on the local policy/V network and apply them to shared memory entropy = self.apply_update(states, actions, targets, advantages) elapsed_time = time.time() - self.start_time steps_per_sec = self.global_step.value() / elapsed_time perf = "{:.0f}".format(steps_per_sec) logger.info( "T{} / EPISODE {} / STEP {}k / REWARD {} / {} STEPS/s".format( self.actor_id, self.local_episode, self.global_step.value() / 1000, total_episode_reward, perf)) self.log_summary(total_episode_reward, np.array(values).mean(), entropy)
def load_and_cache_examples(args, task_name, tokenizer, data_type="train"): if args.local_rank not in [-1, 0] and not evaluate: torch.distributed.barrier() processor = processors[task_name]() # load data features from cache or datafile cached_features_file = os.path.join( args.data_dir, "cache_crf-{}_{}_{}_{}".format( data_type, list(filter(None, args.model_name_or_path.split("/"))).pop(), str(args.train_max_seq_length if data_type == "train" else args.eval_max_seq_length), str(task_name))) if os.path.exists(cached_features_file) and not args.overwrite_cache: logger.info("Loading features from cache file %s", cached_features_file) features = torch.load(cached_features_file) else: logger.info("Creating features from dateset file at %s", cached_features_file) label_list = processor.get_labels() if data_type == "train": examples = processor.get_train_examples(args.data_dir) elif data_type == "dev": examples = processor.get_dev_examples(args.data_dir) else: examples = processor.get_test_examples(args.data_dir) features = convert_examples_to_features( examples=examples, tokenizer=tokenizer, label_list=label_list, max_seq_length=args.train_max_seq_length if data_type == "train" else args.eval_max_seq_length, cls_token_at_end=bool(args.model_type in ["xlnet"]), pad_on_left=bool(args.model_type in ["xlnet"]), cls_token=tokenizer.cls_token, cls_token_segment_id=2 if args.model_type in ["xlnet"] else 0, sep_token=tokenizer.sep_token, #pad on left for xlnet pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token ])[0], pad_token_segment_id=4 if args.model_type in ["xlnet"] else 0, ) if args.local_rank in [-1, 0]: logger.info("save features into cached file %s", cached_features_file) torch.save(features, cached_features_file) if args.local_rank == 0 and not evaluate: torch.distributed.barrier() #convert to tensors and build dateset print(features[0]) all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) all_input_masks = torch.tensor([f.input_mask for f in features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_id for f in features], dtype=torch.long) all_label_ids = torch.tensor([f.label_ids for f in features], dtype=torch.long) all_lens = torch.tensor([f.input_len for f in features], dtype=torch.long) dataset = TensorDataset(all_input_ids, all_input_masks, all_segment_ids, all_lens, all_label_ids) return dataset
def _tb_load_manifest(file_path, num_labels=31, name_labels=None, name_paths=None, mode='single', ext_data=False, fl_balance=False, r_seed=-1): if not file_path.exists(): logger.error(f"manifest file {file_path} not found.") raise RuntimeError logger.debug(f"loading dataset manifest {file_path} ...") df = pd.read_csv(str(file_path)).fillna(0) if (not ext_data) and (True): # using the clean-set # cleanset if True: ## MGH validation set df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)] if r_seed != -1: df = df.sample(n=1000, replace=True, random_state=r_seed) if (False): df = df.loc[ (df['Hilar/mediastinum>Cardiomegaly>.'] == 1) | (df['Lung density>Increased lung density>Atelectasis'] == 1 ) | (df['Lung density>Increased lung density>Pulmonary edema'] == 1) | (df['Lung density>Increased lung density>pneumonia'] == 1) | (df['Pleura>Pleural effusion>.'] == 1)] df.reset_index(drop=True, inplace=True) ## MGH testset #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][0:250] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][250:500] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][500:750] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][750:] ## CheXpert trainset #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][0:250] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][250:500] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][500:750] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][750:1000] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][1000:1250] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][1250:1500] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][1500:1750] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][1750:2000] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][2000:2250] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][2250:2500] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][2500:2750] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][2750:3000] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][3000:3250] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][3250:3500] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][3500:3750] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][3750:4000] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][4000:4250] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][4250:4500] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][4500:] ## NIH trainset #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][0:500] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][500:1000] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][1000:1500] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][1500:2000] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][2000:2500] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][2500:3000] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][3000:3500] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][3500:4000] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][4000:] ## MIMIC trainset #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][0:500] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][500:1000] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][1000:1500] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][1500:2000] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][2000:2500] #df = df.loc[(df['bad_age'] == 0) & (df['bad_quality'] == 0)][2500:] if (Clean_Neg): #hilar area special care for cl_feature, cl_file in Clean_Neg_list: df_case = pd.read_csv(MGH_DATA_BASE.joinpath( 'clean_nagative_data_v5_deblank/' + cl_file), names=['ACC']) #df_case = pd.read_csv(MGH_DATA_BASE.joinpath('clean_nagative_data_v5/'+cl_file)) #df_case = pd.read_csv(MGH_DATA_BASE.joinpath('clean_negative_data_v5_321/'+cl_file), names=['ACC']) df[f'{cl_feature}'] = df[f'{cl_feature}'].replace(0, -2) #df.loc[df.AccessionNumber.isin(df_case.ACC), f'{cl_feature}'] = 0 df.loc[(df.AccessionNumber.isin(df_case.ACC)) & (df[f'{cl_feature}'] == -2), f'{cl_feature}'] = 0 if (fl_balance): for k, feature in enumerate(label_name): num_p = df.loc[(df[f'{feature}'] == 1)].shape[0] num_n = df.loc[(df[f'{feature}'] == 0)].shape[0] ratio_pn = num_p / num_n ratio_th = 5 if (ratio_pn < (1.0 / ratio_th)): df[f'{feature}'] = df[f'{feature}'].replace(0, -1) df_n = df.loc[(df[f'{feature}'] == -1)].sample( n=(num_p * ratio_th), random_state=2020) df[f'{feature}'].loc[df['AccessionNumber'].isin( df_n['AccessionNumber'])] = 0 pos = df[f'{feature}'].loc[df[f'{feature}'] == 1].shape[0] neg = df[f'{feature}'].loc[df[f'{feature}'] == 0].shape[0] dontcare = df[f'{feature}'].loc[df[f'{feature}'] == -1].shape[0] logger.info( f'[{k:02d}-{feature}] pos: {pos}, neg: {neg}, dont-care: {dontcare}' ) if name_labels == None: df = df[~(df.iloc[:, -(num_labels + 1):-1] == -1).all(1)] else: df = df[~(df[name_labels] == -1).all(1)] df.reset_index(drop=True, inplace=True) if (Clean_Neg): for cl_feature, cl_file in Clean_Neg_list: df[f'{cl_feature}'] = df[f'{cl_feature}'].replace(-2, -1) if False: for k, feature in enumerate(label_name): num_p = df.loc[(df[f'{feature}'] == 1)].shape[0] num_n = df.loc[(df[f'{feature}'] == 0)].shape[0] num_i = df.loc[(df[f'{feature}'] == -1)].shape[0] print( f'{feature}-{num_p}-{num_p/df.shape[0]}-{num_i}-{num_i/df.shape[0]}-{num_n}-{num_n/df.shape[0]}' ) exit(-1) if (True): # in order to add clinical information to network df['ScaledSex'] = df.sex.replace(0, -1) weight_gender = 10 weight_age = 100 min_age = 11.0 max_age = 100.0 #df.PatientAge = (df.PatientAge-min(df.PatientAge))/(max(df.PatientAge)-min(df.PatientAge)) df['ScaledAge'] = (df.PatientAge - min_age) / (max_age - min_age) df.ScaledAge = weight_age * (df.ScaledAge - 0.5) df['ScaledSex'] = weight_gender * df.ScaledSex df.reset_index(drop=True, inplace=True) else: try: df['ScaledSex'] = df.sex.replace(0, -1) weight_gender = 10 weight_age = 100 min_age = 11.0 max_age = 117.0 #df.PatientAge = (df.PatientAge-min(df.PatientAge))/(max(df.PatientAge)-min(df.PatientAge)) df['ScaledAge'] = (df.PatientAge - min_age) / (max_age - min_age) df.ScaledAge = weight_age * (df.ScaledAge - 0.5) df['ScaledSex'] = weight_gender * df.ScaledSex except: df['ScaledAge'] = 0 df['ScaledSex'] = 0 if (mode == 'single') | (mode == 'extd'): LABELS = df.columns[-(num_labels + 1):-1] if name_labels == None else name_labels labels = df[LABELS].astype(int) paths = df['PATH'] if name_paths == None else df[name_paths] ages = df['ScaledAge'].astype(float) genders = df['ScaledSex'].astype(float) df_tmp = pd.concat([paths, ages, genders, labels], axis=1) elif mode == 'double': LABELS = df.columns[-(num_labels + 2):-2] if name_labels == None else name_labels labels = df[LABELS].astype(int) paths = df[df.columns[-2:]] if name_paths == None else df[name_paths] df_tmp = pd.concat([paths, labels], axis=1) else: raise RuntimeError entries = df_tmp logger.debug(f"{len(entries)} entries are loaded.") return entries
def __init__(self, params: dict, dataset: LmSeqsDataset, token_probs: torch.tensor, student: nn.Module, teacher: nn.Module): logger.info('Initializing Distiller') self.params = params self.dump_path = params.dump_path self.multi_gpu = params.multi_gpu self.fp16 = params.fp16 self.student = student self.teacher = teacher self.student_config = student.config self.vocab_size = student.config.vocab_size if params.n_gpu <= 1: sampler = RandomSampler(dataset) else: sampler = DistributedSampler(dataset) if params.group_by_size: groups = create_lengths_groups(lengths=dataset.lengths, k=params.max_model_input_size) sampler = GroupedBatchSampler(sampler=sampler, group_ids=groups, batch_size=params.batch_size) else: sampler = BatchSampler(sampler=sampler, batch_size=params.batch_size, drop_last=False) self.dataloader = DataLoader(dataset=dataset, batch_sampler=sampler, collate_fn=dataset.batch_sequences) self.temperature = params.temperature assert self.temperature > 0. self.alpha_ce = params.alpha_ce self.alpha_mlm = params.alpha_mlm self.alpha_clm = params.alpha_clm self.alpha_mse = params.alpha_mse self.alpha_cos = params.alpha_cos self.mlm = params.mlm if self.mlm: logger.info(f'Using MLM loss for LM step.') self.mlm_mask_prop = params.mlm_mask_prop assert 0.0 <= self.mlm_mask_prop <= 1.0 assert params.word_mask + params.word_keep + params.word_rand == 1.0 self.pred_probs = torch.FloatTensor( [params.word_mask, params.word_keep, params.word_rand]) self.pred_probs = self.pred_probs.to( f'cuda:{params.local_rank}' ) if params.n_gpu > 0 else self.pred_probs self.token_probs = token_probs.to( f'cuda:{params.local_rank}' ) if params.n_gpu > 0 else token_probs if self.fp16: self.pred_probs = self.pred_probs.half() self.token_probs = self.token_probs.half() else: logger.info(f'Using CLM loss for LM step.') self.epoch = 0 self.n_iter = 0 self.n_total_iter = 0 self.n_sequences_epoch = 0 self.total_loss_epoch = 0 self.last_loss = 0 self.last_loss_ce = 0 self.last_loss_mlm = 0 self.last_loss_clm = 0 if self.alpha_mse > 0.: self.last_loss_mse = 0 if self.alpha_cos > 0.: self.last_loss_cos = 0 self.last_log = 0 self.ce_loss_fct = nn.KLDivLoss(reduction='batchmean') self.lm_loss_fct = nn.CrossEntropyLoss(ignore_index=-1) if self.alpha_mse > 0.: self.mse_loss_fct = nn.MSELoss(reduction='sum') if self.alpha_cos > 0.: self.cosine_loss_fct = nn.CosineEmbeddingLoss(reduction='mean') logger.info('--- Initializing model optimizer') assert params.gradient_accumulation_steps >= 1 self.num_steps_epoch = len(self.dataloader) num_train_optimization_steps = int( self.num_steps_epoch / params.gradient_accumulation_steps * params.n_epoch) + 1 no_decay = ['bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [{ 'params': [ p for n, p in student.named_parameters() if not any(nd in n for nd in no_decay) and p.requires_grad ], 'weight_decay': params.weight_decay }, { 'params': [ p for n, p in student.named_parameters() if any(nd in n for nd in no_decay) and p.requires_grad ], 'weight_decay': 0.0 }] logger.info( "------ Number of trainable parameters (student): %i" % sum([ p.numel() for p in self.student.parameters() if p.requires_grad ])) logger.info("------ Number of parameters (student): %i" % sum([p.numel() for p in self.student.parameters()])) self.optimizer = AdamW(optimizer_grouped_parameters, lr=params.learning_rate, eps=params.adam_epsilon, betas=(0.9, 0.98)) warmup_steps = math.ceil(num_train_optimization_steps * params.warmup_prop) self.scheduler = get_linear_schedule_with_warmup( self.optimizer, num_warmup_steps=warmup_steps, num_training_steps=num_train_optimization_steps) if self.fp16: try: from apex import amp except ImportError: raise ImportError( "Please install apex from https://www.github.com/nvidia/apex to use fp16 training." ) logger.info( f"Using fp16 training: {self.params.fp16_opt_level} level") self.student, self.optimizer = amp.initialize( self.student, self.optimizer, opt_level=self.params.fp16_opt_level) self.teacher = self.teacher.half() if self.multi_gpu: if self.fp16: from apex.parallel import DistributedDataParallel logger.info( "Using apex.parallel.DistributedDataParallel for distributed training." ) self.student = DistributedDataParallel(self.student) else: from torch.nn.parallel import DistributedDataParallel logger.info( "Using nn.parallel.DistributedDataParallel for distributed training." ) self.student = DistributedDataParallel( self.student, device_ids=[params.local_rank], output_device=params.local_rank, find_unused_parameters=True) self.is_master = params.is_master if self.is_master: logger.info('--- Initializing Tensorboard') self.tensorboard = SummaryWriter( log_dir=os.path.join(self.dump_path, 'log', 'train')) self.tensorboard.add_text(tag='config/training', text_string=str(self.params), global_step=0) self.tensorboard.add_text(tag='config/student', text_string=str(self.student_config), global_step=0)
def startGame(): ''' Scripts to start benchmarking ''' exeFile = r'{STEAM_DIRECTORY}//{GAME_DIRECTORY}//GlimpseGame//Binaries//Win64//{GAME_EXECUTOR}'.format( STEAM_DIRECTORY=STEAM_DIRECTORY, GAME_DIRECTORY=GAME_DIRECTORY, GAME_EXECUTOR=GAME_EXECUTOR) ## Start game launcher # - return 0 and end the whole process, if failed # - otherwise, keep running the process tries = 10 while tries != 0: logger.info("Opening Game Launcher") startGame = win32api.ShellExecute(1, 'open', exeFile, '', '', 1) if tries == 1 and not startGame: screenShootName = utils.screen.saveScreenShoot( GAME_NAME, "OpenLauncherFailed") logger.error( 'Opening Game Launcher Failed! Screenshoot Created: %s' % screenShootName) print( "****** Failed to open Game Launcher!!! Process stopped ******\n" ) return 0 if startGame: logger.info("Open Game Launcher Succeed") print("Open Game Launcher Succeed!!") break else: tries -= 1 time.sleep(1) time.sleep(10) # ## Apply ENTER on the launcher to start game # # return 0, if failed to apply ENTER key on he launcher # # - otherwise, keep running the process # tries = 0 # while utils.screen.findWindow(GAME_NAME): # logger.info('Opening Game: %s'%GAME_NAME) # utils.input.clickLeft(1310, 385) # time.sleep(10) # gameHD = utils.screen.findWindow("We Happy Few (64-bit, PCD3D_SM5)") # if gameHD: # tries = 0 # break # elif tries > 10: # screenShootName=utils.screen.saveScreenShoot(GAME_NAME, "OpenGameFailed") # logger.error('Opening Game Failed! Screenshoot Created: %s'%screenShootName) # print("****** Failed to open Game!!! Process stopped ******\n") # return 0 # tries += 1 # time.sleep(3) logger.info(_TAB + 'Waiting for game to start') ## Give 25 sec for the game to start print("Waiting for game to start...") time.sleep(60) #################################################################################### # Start Game loop = LOOP_TIMES while (loop != 0): time.sleep(5) # Skip Press button to start tmp = 10 while (tmp != 0): time.sleep(0.5) tmp = tmp - 1 utils.input.clickLeft(960, 540) time.sleep(20) utils.keyboardUtils.callTinyTask("enter") time.sleep(10) utils.keyboardUtils.callTinyTask("enter") logger.info(_TAB + 'Starting Testing') print("Start Testing...") ## Perform random Character control for 5 min utils.keyboardUtils.randomCharacterControl(300) if loop == -1: break else: loop -= 1 logger.info('Loop times remained: %s' % loop) print("Loop times remained: %s\n" % loop) logger.info(_TAB + 'All Loop Finishbed') print("Finished!") #################################################################################### # Quit Game time.sleep(10) # utils.keyboardUtils.press_alt_f4() utils.input.key_alt_f4() return startGame
def normalize_message(message): normalized = '' entity_text = '' entity_normalized = '' offsets = [0] lengths = [0] # Encodes the judged message to preserve possible emojis msg_enc = message.text.encode('utf-16le') # Strips the message in chunks: from entity to entity for entity in message.entities: # Keeps track of the entities position in the message offsets.append(entity.offset) lengths.append(entity.length) # Decodes a chunk of the message to get contents of an entity # NOTE that in the encoded message every symbols comes with a pair, hence doubling the indices entity_text = msg_enc[2 * offsets[-1]:2 * (offsets[-1] + lengths[-1])].decode('utf-16le') # Strips the found link and other entities # rewrites it in the Markdown-friendly format if entity.type == 'text_link': # Detects a hidden link entity_normalized = '[{1}]({0})'.format(entity.url, entity_text) elif entity.type == 'url': # Detects an ordinary link entity_normalized = '[{0}]({0})'.format(entity_text) elif entity.type == 'email': # Detects an email entity_normalized = '[{0}](mailto:{0})'.format(entity_text) elif entity.type == 'mention': # Detects a user/chat mention entity_normalized = '[@{0}](https://t.me/{0})'.format( entity_text[1:]) elif entity.type == 'bold': # Detects a bolded text entity_normalized = '*{}*'.format(entity_text) elif entity.type == 'italic': # Detects an italicized text entity_normalized = '_{}_'.format(entity_text) elif entity.type == 'code': # Detects a coded text entity_normalized = '`{}`'.format(entity_text) elif entity.type == 'pre': # Detects a prettified text entity_normalized = '```{}```'.format(entity_text) else: # Detects other entities logger.info('Unrecognized entity of type {0}, offset {1} and length {2} detected. '\ 'Adding as text'.format(entity.type, entity.offset, entity.length)) entity_normalized = entity_text # Adds the stripped chunk to the final string normalized += msg_enc[2 * (offsets[-2] + lengths[-2]):2 * offsets[-1]].decode( 'utf-16le') + entity_normalized # Adds the part after the last entity normalized += msg_enc[2 * (offsets[-1] + lengths[-1]):].decode('utf-16le') return normalized
def train(self): """ Main actor learner loop for advantage actor critic learning. """ logger.debug("Actor {} resuming at Step {}".format( self.actor_id, self.global_step.value())) s = self.emulator.get_initial_state() total_episode_reward = 0 s_batch = [] a_batch = [] y_batch = [] adv_batch = [] seq_len_batch = [] reset_game = False episode_over = False start_time = time.time() steps_at_last_reward = self.local_step while (self.global_step.value() < self.max_global_steps): # Sync local learning net with shared mem self.sync_net_with_shared_memory(self.local_network, self.learning_vars) self.save_vars() local_step_start = self.local_step rewards = [] states = [] actions = [] values = [] seq_lengths = [] while not (episode_over or (self.local_step - local_step_start == self.max_local_steps)): # Choose next action and execute it action_sequence, readout_v_t = self.sample_action_sequence(s) # if self.is_master() and (self.local_step % 100 == 0): # logger.debug("pi={}, V={}".format(readout_pi_t, readout_v_t)) acc_reward = 0.0 length = 0 for action in action_sequence: length += 1 a = np.argmax(action) if a == self.num_actions or episode_over: break new_s, reward, episode_over = self.emulator.next( action[:self.num_actions]) acc_reward += reward reward = acc_reward if reward != 0.0: steps_at_last_reward = self.local_step total_episode_reward += reward # Rescale or clip immediate reward reward = self.rescale_reward(reward) rewards.append(reward) seq_lengths.append(length) states.append(s) actions.append(action_sequence) values.append(readout_v_t) s = new_s self.local_step += 1 self.global_step.increment() if self.local_step % 1000 == 0: pass # Pdb().set_trace() # Calculate the value offered by critic in the new state. if episode_over: R = 0 else: R = self.session.run( self.local_network.output_layer_v, feed_dict={self.local_network.input_ph: [new_s]})[0][0] sel_actions = [] for i in reversed(range(len(states))): R = rewards[i] + self.gamma * R y_batch.append(R) a_batch.append(actions[i]) s_batch.append(states[i]) adv_batch.append(R - values[i]) seq_len_batch.append(seq_lengths[i]) sel_actions.append(np.argmax(actions[i])) padded_output_sequences = np.array([ np.vstack([ seq[:length, :], np.zeros( (max(seq_len_batch) - length, self.num_actions + 1)) ]) for length, seq in zip(seq_len_batch, a_batch) ]) go_input = np.zeros((len(s_batch), 1, self.num_actions + 1)) go_input[:, :, self.num_actions] = 1 padded_input_sequences = np.hstack( [go_input, padded_output_sequences[:, :-1, :]]) print('Sequence lengths:', seq_lengths) print('Actions:', [np.argmax(a) for a in a_batch[0]]) allowed_actions = np.ones( (len(s_batch), max(seq_len_batch), self.num_actions + 1)) allowed_actions[:, 0, -1] = 0 #empty sequence is not a valid action feed_dict = { self.local_network.input_ph: s_batch, self.local_network.critic_target_ph: y_batch, self.local_network.adv_actor_ph: adv_batch, self.local_network.decoder_initial_state: np.zeros((len(s_batch), self.local_network.decoder_hidden_state_size * 2)), self.local_network.action_inputs: padded_input_sequences, self.local_network.action_outputs: padded_output_sequences, self.local_network.allowed_actions: allowed_actions, self.local_network.use_fixed_action: True, self.local_network.decoder_seq_lengths: seq_lengths, self.local_network.temperature: 1.0, } entropy, grads = self.session.run( [ self.local_network.entropy, # self.local_network.adv_critic, # self.local_network.output_layer_v, self.local_network.get_gradients ], feed_dict=feed_dict) print('Entropy:', entropy) #, 'Adv:', advantage #, 'Value:', value self.apply_gradients_to_shared_memory_vars(grads) s_batch = [] a_batch = [] y_batch = [] adv_batch = [] seq_len_batch = [] # prevent the agent from getting stuck if (self.local_step - steps_at_last_reward > 5000 or (self.emulator.get_lives() == 0 and self.emulator.game not in ONE_LIFE_GAMES)): steps_at_last_reward = self.local_step episode_over = True reset_game = True # Start a new game on reaching terminal state if episode_over: elapsed_time = time.time() - start_time global_t = self.global_step.value() steps_per_sec = global_t / elapsed_time perf = "{:.0f}".format(steps_per_sec) logger.info( "T{} / STEP {} / REWARD {} / {} STEPS/s, Actions {}". format(self.actor_id, global_t, total_episode_reward, perf, sel_actions)) self.vis.plot_current_errors(global_t, total_episode_reward) self.log_summary(total_episode_reward, entropy) episode_over = False total_episode_reward = 0 steps_at_last_reward = self.local_step if reset_game or self.emulator.game in ONE_LIFE_GAMES: s = self.emulator.get_initial_state() reset_game = False
args=(i, cls.url, sleep))) workers[-1].start() for th in workers: th.join() @classmethod def test_async_sleep_url(cls, n, sleep=1): workers = [] for i in range(n): workers.append( threading.Thread(target=cls.one_request, args=(i, cls.async_url, sleep))) workers[-1].start() for th in workers: th.join() @classmethod def one_request(cls, tid, url, sleep): rs = requests.get(url=url, params={'ts': sleep}) logger.info('thread: {}, response: {}'.format(tid, rs.json())) if __name__ == '__main__': import time tic = time.time() MultiThreadsClient.test_sleep_url(5, 1) logger.info('sleep url: {}'.format(time.time() - tic)) tic = time.time() MultiThreadsClient.test_async_sleep_url(5, 1) logger.info('sleep url: {}'.format(time.time() - tic))
def predict(args, model, tokenizer, prefix=""): pred_output_dir = args.output_dir if not os.path.exists(pred_output_dir) and args.local_rank in [-1, 0]: os.makedirs(pred_output_dir) test_dataset = load_and_cache_examples(args, args.task_name, tokenizer, data_type='test') # Note that DistributedSampler samples randomly test_sampler = SequentialSampler( test_dataset) if args.local_rank == -1 else DistributedSampler( test_dataset) test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=1, collate_fn=collate_fn) # Eval! logger.info("***** Running prediction %s *****", prefix) logger.info(" Num examples = %d", len(test_dataset)) logger.info(" Batch size = %d", 1) results = [] output_predict_file = os.path.join(pred_output_dir, prefix, "test_prediction.json") pbar = ProgressBar(n_total=len(test_dataloader), desc="Predicting") if isinstance(model, nn.DataParallel): model = model.module for step, batch in enumerate(test_dataloader): model.eval() batch = tuple(t.to(args.device) for t in batch) with torch.no_grad(): inputs = { "input_ids": batch[0], "attention_mask": batch[1], "labels": None, 'input_lens': batch[4] } if args.model_type != "distilbert": # XLM and RoBERTa don"t use segment_ids inputs["token_type_ids"] = (batch[2] if args.model_type in ["bert", "xlnet"] else None) outputs = model(**inputs) logits = outputs[0] tags = model.crf.decode(logits, inputs['attention_mask']) tags = tags.squeeze(0).cpu().numpy().tolist() preds = tags[0][1:-1] # [CLS]XXXX[SEP] label_entities = get_entities(preds, args.id2label, args.markup) json_d = {} json_d['id'] = step json_d['tag_seq'] = " ".join([args.id2label[x] for x in preds]) json_d['entities'] = label_entities results.append(json_d) pbar(step) logger.info("\n") with open(output_predict_file, "w") as writer: for record in results: writer.write(json.dumps(record) + '\n') if args.task_name == 'cluener': output_submit_file = os.path.join(pred_output_dir, prefix, "test_submit.json") test_text = [] with open(os.path.join(args.data_dir, "test.json"), 'r') as fr: for line in fr: test_text.append(json.loads(line)) test_submit = [] for x, y in zip(test_text, results): json_d = {} json_d['id'] = x['id'] json_d['label'] = {} entities = y['entities'] words = list(x['text']) if len(entities) != 0: for subject in entities: tag = subject[0] start = subject[1] end = subject[2] word = "".join(words[start:end + 1]) if tag in json_d['label']: if word in json_d['label'][tag]: json_d['label'][tag][word].append([start, end]) else: json_d['label'][tag][word] = [[start, end]] else: json_d['label'][tag] = {} json_d['label'][tag][word] = [[start, end]] test_submit.append(json_d) json_to_text(output_submit_file, test_submit)
def one_request(cls, tid, url, sleep): rs = requests.get(url=url, params={'ts': sleep}) logger.info('thread: {}, response: {}'.format(tid, rs.json()))
def main(): args = get_argparse().parse_args() # 创建输出路径 if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) args.output_dir = args.output_dir + "{}".format(args.model_type) if not os.path.exists(args.output_dir): os.mkdir(args.output_dir) time_ = time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime()) # 定义logger 文件 init_logger(log_file=args.output_dir + f"/{args.model_type}-{args.task_name}-{time_}.log") if os.path.exists(args.output_dir) and os.listdir( args.output_dir ) and args.do_train and not args.overwrite_output_dir: raise ValueError( "Output directory ({}) is already exists and is not empty.use --overwrite_output_dir to overcome." .format(args.output_dir)) # setup distant debugging if needed if args.server_ip and args.server_port: # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script import ptvsd print("waiting for debugger attach") ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True) ptvsd.wait_for_attach() # setup cuda, gpu & distributed training if args.local_rank == -1 or args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") args.n_gpu = torch.cuda.device_count() else: torch.cuda.set_device(args.local_rank) device = torch.device("cuda", args.local_rank) torch.distributed.init_process_group(backend="nccl") args.n_gpu = 1 args.device = device logger.warning( "Process rank: %s,device: %s ,n_gpus: %s,distributed training: %s,16-bits training: %s", args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16, ) # set seed seed_everyting(args.seed) # prepare ner task args.task_name = args.task_name.lower() if args.task_name not in processors: raise ValueError("Task not found: %s" % (args.task_name)) processor = processors[args.task_name]() label_list = processor.get_labels() args.id2label = {i: label for i, label in enumerate(label_list)} args.label2id = {label: i for i, label in enumerate(label_list)} num_labels = len(label_list) # load pretrained model and tokenizer if args.local_rank not in [-1, 0]: torch.distributed.barrier() args.model_type = args.model_type.lower() config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] config = config_class.from_pretrained( args.config_name if args.config_name else args.model_name_or_path, num_labels=num_labels, cache_dir=args.cache_dir if args.cache_dir else None) tokenizer = tokenizer_class.from_pretrained( args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case, cache_dir=args.cache_dir if args.cache_dir else None) model = model_class.from_pretrained( args.model_name_or_path, from_tf=bool(".ckpt" in args.model_name_or_path), config=config, cache_dir=args.cache_dir if args.cache_dir else None) if args.local_rank == 0: torch.distributed.barrier() model.to(args.device) logger.info("Training/evaluation parameters %s", args) #Training if args.do_train: train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, data_type="train") global_step, tr_loss = train(args, train_dataset, model, tokenizer) logger.info("golbal_step = %s,average loss= %s", global_step, tr_loss) # saving best-practice : if you use default names for the model,you can reload it using from_pytrained if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0): # create output directory if needed if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: os.makedirs(args.output_dir) logger.info("Saving model checkpoint in %s", args.output_dir) model_to_save = (model.module if model is hasattr(model, "module") else model) model_to_save.save_pretrained(args.output_dir) tokenizer.save_vocabulary(args.output_dir) torch.save(args, os.path.join(args.output_dir, "training_args.bin")) # Evaluation results = {} if args.do_eval and args.local_rank in [-1, 0]: tokenizer = tokenizer_class.from_pretrained( args.output_dir, do_lower_case=args.do_lower_case) checkpoints = [args.output_dir] if args.eval_all_checkpoints: checkpoints = list( os.path.dirname(c) for c in sorted( glob.glob(args.output_dir + "/**/" + WEIGHTS_NAME, recursive=True))) logging.getLogger("pytorch_transformers.modeling_utils").setLevel( logging.WARNING) logger.info("Evaluate the following checkpoints: %s", checkpoints) for checkpoint in checkpoints: global_step = checkpoint.split( "-")[-1] if len(checkpoints) > 1 else "" prefix = checkpoint.split( "/")[-1] if checkpoint.find("checkpoint") != -1 else "" model = model_class.from_pretrained(checkpoint, config=config) model.to(args.device) result = evaluate(args, model, tokenizer, prefix=prefix) if global_step: result = { "{}_{}".format(global_step, k): v for k, v in result.items() } results.update(result) output_eval_file = os.path.join(args.output_dir, "eval_results.txt") with open(output_eval_file, "w") as writer: for key in sorted(results.keys()): writer.write("{} = {}\n".format(key, str(results[key]))) # predict if args.do_predict and args.local_rank in [-1, 0]: tokenizer = tokenizer_class.from_pretrained( args.output_dir, do_lower_case=args.do_lower_case) checkpoints = [args.output_dir] if args.predict_checkpoints > 0: checkpoints = list( os.path.dirname(c) for c in sorted( glob.glob(args.output_dir + "/**/" + WEIGHTS_NAME, recursive=True))) logging.getLogger("transformers.modeling_utils").setLevel( logging.WARNING) checkpoints = [ x for x in checkpoints if x.split("-")[-1] == str(args.predict_checkpoints) ] for checkpoint in checkpoints: prefix = checkpoint.split( "/")[-1] if checkpoint.find("checkpoint") != -1 else "" model = model_class.from_pretrained(checkpoint, config=config) model.to(args.device) predict(args, model, tokenizer, prefix)
def test(rank, args, shared_model, gl_step_cnt): torch.manual_seed(args.seed + rank) env = create_atari_env(args.env_name) env.seed(args.seed + rank) model = ActorCritic(env.observation_space.shape[0], env.action_space) model.eval() state = env.reset() state = torch.from_numpy(state) reward_sum = 0 done = True start_time = time.time() local_episode_num = 0 # a quick hack to prevent the agent from stucking actions = deque(maxlen=100) episode_length = 0 while True: episode_length += 1 # Sync with the shared model if done: model.load_state_dict(shared_model.state_dict()) cx = Variable(torch.zeros(1, 256), volatile=True) hx = Variable(torch.zeros(1, 256), volatile=True) else: cx = Variable(cx.data, volatile=True) hx = Variable(hx.data, volatile=True) value, logit, (hx, cx) = model((Variable(state.unsqueeze(0), volatile=True), (hx, cx))) prob = F.softmax(logit) action = prob.max(1)[1].data.numpy() state, reward, done, _ = env.step(action[0, 0]) done = done or episode_length >= args.max_episode_length reward_sum += reward # a quick hack to prevent the agent from stucking actions.append(action[0, 0]) if actions.count(actions[0]) == actions.maxlen: done = True if done: passed_time = time.time() - start_time local_episode_num += 1 global_step_count = gl_step_cnt.get_value() logger.info("Time {}, episode reward {}, episode length {}".format( time.strftime("%Hh %Mm %Ss", time.gmtime(passed_time)), reward_sum, episode_length)) tb.log_value('steps_second', global_step_count / passed_time, global_step_count) tb.log_value('reward', reward_sum, global_step_count) reward_sum = 0 episode_length = 0 actions.clear() state = env.reset() time.sleep(60) state = torch.from_numpy(state)
def train(self, sess, summary_writer, data_A, data_B): logger.info('Start training.') logger.info(' {} images from A'.format(len(data_A))) logger.info(' {} images from B'.format(len(data_B))) data_size = min(len(data_A), len(data_B)) num_batch = data_size // self._batch_size epoch_length = num_batch * self._batch_size num_initial_iter = 10 num_decay_iter = 10 lr = lr_initial = 0.0002 lr_decay = lr_initial / num_decay_iter history_a = HistoryQueue(shape=self._image_shape, size=50) history_b = HistoryQueue(shape=self._image_shape, size=50) initial_step = sess.run(self.global_step) num_global_step = (num_initial_iter + num_decay_iter) * epoch_length t = trange(initial_step, num_global_step, total=num_global_step, initial=initial_step) for step in t: #TODO: resume training with global_step epoch = step // epoch_length iter = step % epoch_length if epoch > num_initial_iter: lr = max(0.0, lr_initial - (epoch - num_initial_iter) * lr_decay) if iter == 0: random.shuffle(data_A) random.shuffle(data_B) image_a = np.stack(data_A[iter * self._batch_size:(iter + 1) * self._batch_size]) image_b = np.stack(data_B[iter * self._batch_size:(iter + 1) * self._batch_size]) fake_a, fake_b = sess.run([self.image_ba, self.image_ab], feed_dict={ self.image_a: image_a, self.image_b: image_b, self.is_train: True }) fake_a = history_a.query(fake_a) fake_b = history_b.query(fake_b) fetches = [ self.loss_D_a, self.loss_D_b, self.loss_G_ab, self.loss_G_ba, self.loss_cycle, self.optimizer_D_a, self.optimizer_D_b, self.optimizer_G_ab, self.optimizer_G_ba ] if step % self._log_step == 0: fetches += [self.summary_op] fetched = sess.run(fetches, feed_dict={ self.image_a: image_a, self.image_b: image_b, self.is_train: True, self.lr: lr, self.history_fake_a: fake_a, self.history_fake_b: fake_b }) if step % self._log_step == 0: summary_writer.add_summary(fetched[-1], step) summary_writer.flush() t.set_description( 'Loss: D_a({:.3f}) D_b({:.3f}) G_ab({:.3f}) G_ba({:.3f}) cycle({:.3f})' .format(fetched[0], fetched[1], fetched[2], fetched[3], fetched[4]))
def train(model_class): """Train neural network for a number of steps.""" logger.info("\nstart training...") with tf.Graph().as_default(): # build computing graph with tf.variable_scope("model", reuse=None): model_train = model_class(is_train=True) if FLAGS.eval_step > 0: with tf.variable_scope("model", reuse=True): model_eval = model_class(is_train=False) saver = tf.train.Saver(tf.all_variables(), max_to_keep=FLAGS.num_checkpoints) sv = tf.train.Supervisor(logdir=RESULT_DIR, saver=saver, save_summaries_secs=0, save_model_secs=0) logger.newline() logger.info("start building Graph (This might take a while)") # Start running operations on the Graph. sess = sv.prepare_or_wait_for_session(config=tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement)) logger.newline() logger.info("start training...") try: while not sv.should_stop(): start_time = time.time() step, loss_value, top_k = model_train.train_step(sess) duration = time.time() - start_time assert not np.isnan( loss_value), 'Model diverged with loss = NaN' # print current state if step % FLAGS.print_step == 0: num_examples_per_step = FLAGS.batch_size examples_per_sec = num_examples_per_step / duration sec_per_batch = float(duration) precision = np.sum(top_k) / FLAGS.batch_size format_str = ( 'step %d, loss = %.2f, precision = %.2f (%.1f ' 'examples/sec; %.3f sec/batch)') logger.info(format_str % (step, loss_value, precision, examples_per_sec, sec_per_batch)) # save summary if step % FLAGS.summary_step == 0: summary_str = sess.run(sv.summary_op) sv.summary_writer.add_summary(summary_str, step) logger.info("step: {}, wrote summaries.".format(step)) # Save the model checkpoint periodically and eval on test set. if FLAGS.checkpoint_step > 0 and step % FLAGS.checkpoint_step == 0: saver_path = sv.saver.save(sess, CHECKPOINT_PATH, global_step=step) logger.newline(2) logger.info("Saved model checkpoint to {}\n\n".format( saver_path)) if FLAGS.eval_step > 0 and step % FLAGS.eval_step == 0: logger.newline(2) logger.info("evaluating current model:") precision = model_eval.eval_once(sess) logger.info('%s: precision @ 1 = %.3f' % (time.strftime("%c"), precision)) summary = tf.Summary() summary.ParseFromString(sess.run(sv.summary_op)) summary.value.add(tag='precision @ 1', simple_value=precision) sv.summary_writer.add_summary(summary, step) logger.info("write eval summary\n\n") # sleep for test use if FLAGS.sleep > 0: logger.info("sleep {} second...".format(FLAGS.sleep)) time.sleep(FLAGS.sleep) except tf.errors.OutOfRangeError: logger.info("sv checkpoint saved path: " + sv.save_path) logger.info("Done~\n\n") finally: sv.request_stop() sv.wait_for_stop() sess.close()
def run(self): for res in socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM): af, socktype, proto, cannonname, sa = res try: sock = socket.socket(af, socktype, proto) sock.setblocking(0) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) except socket.error: sock = None continue try: sock.bind(sa) sock.listen(5) except socket.error: sock.close() sock = None continue break host = sa[0] if af == socket.AF_INET6: host = "[%s]" % host if sock is None: print_log( "could not open " + ("SSL" if self.use_ssl else "TCP") + " socket on %s:%d" % (host, self.port)) return print_log( ("SSL" if self.use_ssl else "TCP") + " server started on %s:%d" % (host, self.port)) sock_fd = sock.fileno() poller = select.poll() poller.register(sock) def stop_session(fd): try: # unregister before we close s poller.unregister(fd) except BaseException as e: logger.error('unregister error:' + str(e)) session = self.fd_to_session.pop(fd) # this will close the socket session.stop() def check_do_handshake(session): if session.handshake: return try: session._connection.do_handshake() except ssl.SSLError as err: if err.args[0] == ssl.SSL_ERROR_WANT_READ: return elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE: poller.modify(session.raw_connection, READ_WRITE) return else: raise BaseException(str(err)) poller.modify(session.raw_connection, READ_ONLY) session.handshake = True redo = [] while not self.shared.stopped(): if self.shared.paused(): sessions = self.fd_to_session.keys() if sessions: logger.info("closing %d sessions"%len(sessions)) for fd in sessions: stop_session(fd) time.sleep(1) continue if redo: events = redo redo = [] else: now = time.time() for fd, session in self.fd_to_session.items(): if now - session.time > 0.01 and session.message: cmd = session.parse_message() if not cmd: break if cmd == 'quit': data = False break session.time = now self.handle_command(cmd, session) # check sessions that need to write if session.need_write: poller.modify(session.raw_connection, READ_WRITE) session.need_write = False # collect garbage if now - session.time > session.timeout: stop_session(fd) events = poller.poll(TIMEOUT) for fd, flag in events: # open new session if fd == sock_fd: if flag & (select.POLLIN | select.POLLPRI): try: connection, address = sock.accept() session = TcpSession(self.dispatcher, connection, address, use_ssl=self.use_ssl, ssl_certfile=self.ssl_certfile, ssl_keyfile=self.ssl_keyfile) except BaseException as e: logger.error("cannot start TCP session" + str(e) + ' ' + repr(address)) connection.close() continue connection = session._connection connection.setblocking(False) self.fd_to_session[connection.fileno()] = session poller.register(connection, READ_ONLY) continue # existing session session = self.fd_to_session[fd] s = session._connection # non-blocking handshake try: check_do_handshake(session) except BaseException as e: #logger.error('handshake failure:' + str(e) + ' ' + repr(session.address)) stop_session(fd) continue # anti DOS now = time.time() if now - session.time < 0.01: continue # handle inputs if flag & (select.POLLIN | select.POLLPRI): try: data = s.recv(self.buffer_size) except ssl.SSLError as x: if x.args[0] == ssl.SSL_ERROR_WANT_READ: pass elif x.args[0] == ssl.SSL_ERROR_SSL: pass else: logger.error('SSL recv error:'+ repr(x)) continue except socket.error as x: if x.args[0] != 104: logger.error('recv error: ' + repr(x) +' %d'%fd) stop_session(fd) continue except ValueError as e: logger.error('recv error: ' + str(e) +' %d'%fd) stop_session(fd) continue if data: session.message += data if len(data) == self.buffer_size: redo.append((fd, flag)) if not data: stop_session(fd) continue elif flag & select.POLLHUP: print_log('client hung up', session.address) stop_session(fd) elif flag & select.POLLOUT: # Socket is ready to send data, if there is any to send. if session.retry_msg: next_msg = session.retry_msg else: try: next_msg = session.response_queue.get_nowait() except queue.Empty: # No messages waiting so stop checking for writability. poller.modify(s, READ_ONLY) continue try: sent = s.send(next_msg) except socket.error as x: logger.error("send error:" + str(x)) stop_session(fd) continue session.retry_msg = next_msg[sent:] elif flag & select.POLLERR: print_log('handling exceptional condition for', session.address) stop_session(fd) elif flag & select.POLLNVAL: print_log('invalid request', session.address) stop_session(fd) print_log('TCP thread terminating', self.shared.stopped())
def print_result(message_data, result): """ On success print result """ logger.info( f"The result of message {message_data['message_id']} was {result}.")
def train(self, sess, summary_writer, data_A, data_B): logger.info('Start training.') logger.info(' {} images from A'.format(len(data_A))) logger.info(' {} images from B'.format(len(data_B))) assert len(data_A) == len(data_B), \ 'Data size mismatch dataA(%d) dataB(%d)' % (len(data_A), len(data_B)) data_size = len(data_A) num_batch = data_size // self._batch_size epoch_length = num_batch * self._batch_size num_initial_iter = 8 num_decay_iter = 2 lr = lr_initial = 0.0002 lr_decay = lr_initial / num_decay_iter initial_step = sess.run(self.global_step) num_global_step = (num_initial_iter + num_decay_iter) * epoch_length t = trange(initial_step, num_global_step, total=num_global_step, initial=initial_step) for step in t: #TODO: resume training with global_step epoch = step // epoch_length iter = step % epoch_length if epoch > num_initial_iter: lr = max(0.0, lr_initial - (epoch - num_initial_iter) * lr_decay) # if iter == 0: # #data = zip(data_A, data_B) # random.shuffle(data_A) # random.shuffle(data_B) # #data_A, data_B = zip(*data) ran_choice = np.random.choice(len(data_A), self._batch_size) image_a = data_A[ran_choice] image_b = data_B[ran_choice] sample_z = np.random.normal(size=(self._batch_size, self._latent_dim)) fetches = [self.loss, self.optimizer_D, self.optimizer_G, self.optimizer_E] if step % self._log_step == 0: fetches += [self.summary_op] fetched = sess.run(fetches, feed_dict={self.image_a: image_a, self.image_b: image_b, self.is_train: True, self.lr: lr, self.z: sample_z}) if step % self._log_step == 0: z = np.random.normal(size=(self._batch_size, self._latent_dim)) image_ab = sess.run(self.image_ab, feed_dict={self.image_a: image_a, self.z: z, self.is_train: False}) imsave('results/r_{}.jpg'.format(step), np.squeeze(image_ab[0:1], axis=0)) summary_writer.add_summary(fetched[-1], step) summary_writer.flush() t.set_description('Loss({:.3f})'.format(fetched[0]))
def refresh(self): """ Refreshes the current page """ logger.info("Refreshing the page") self.driver.refresh()
import routes from config import conf from flask import Flask from utils import logger app = Flask(__name__) if __name__ == '__main__': logger.info("Server is starting...") routes.cross_domain(app) routes.settle(app) logger.info(app.url_map) app.run(host='127.0.0.1', port=conf.PORT, debug=False)
def load_embedding(path): rst = np.loadtxt(join(path, 'embedding.txt'), delimiter=' ') rst = np.append(rst, np.zeros(rst.shape[1]).reshape(1, -1), axis=0).astype(np.float32) # add padding vector logger.info("Embedding dim: %d, %d" % (rst.shape[0], rst.shape[1])) return rst
def get_url(self): """ Gets the current URL """ logger.info("Getting the current url") return self.driver.current_url
def train(args, train_dataset, model, tokenizer): args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu) train_sampler = RandomSampler( train_dataset) if args.local_rank == -1 else DistributedSampler( train_dataset) train_dataloder = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size, collate_fn=collate_fn) if args.max_steps > 0: t_total = args.max_steps args.num_train_epochs = args.max_steps // len( train_dataset) // args.gradient_accumulation_steps + 1 else: t_total = len( train_dataset ) * args.num_train_epochs // args.gradient_accumulation_steps #prepare optimizer and schedule (linear warmup and decay) no_decay = ["bias", "LayerNorm.weight"] bert_model_parameters = list(model.bert.named_parameters()) crf_model_parameters = list(model.crf.named_parameters()) linear_model_parameters = list(model.classifier.named_parameters()) optimizer_group_parameters = [{ "params": [ p for n, p in bert_model_parameters if not any([nd in n for nd in no_decay]) ], "weight_decay": args.weight_decay, "lr": args.learning_rate }, { "params": [ p for n, p in bert_model_parameters if any([nd in n for nd in no_decay]) ], "weight_decay": 0.0, "lr": args.learning_rate }, { "params": [ p for n, p in crf_model_parameters if not any([nd in n for nd in no_decay]) ], "weight_decay": args.weight_decay, "lr": args.crf_learning_rate }, { "params": [ p for n, p in crf_model_parameters if any([nd in n for nd in no_decay]) ], "weight_decay": 0.0, "lr": args.crf_learning_rate }, { "params": [ p for n, p in linear_model_parameters if not any([nd in n for nd in no_decay]) ], "weight_decay": args.weight_decay, "lr": args.crf_learning_rate }, { "params": [ p for n, p in linear_model_parameters if any([nd in n for nd in no_decay]) ], "weight_decay": 0.0, "lr": args.crf_learning_rate }] args.warmup_steps = int(t_total * args.warmup_proportion) optimizer = AdamW(optimizer_group_parameters, lr=args.learning_rate, eps=args.adam_epsilon) scheduler = get_linear_schedule_with_warmup( optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) # check if saved optimizer or scheduler if os.path.isfile(os.path.join( args.model_name_or_path, "optimizer.pt")) and os.path.isfile( os.path.join(args.model_name_or_path, "scheduler.pt")): optimizer.load_state_dict( torch.load(os.path.join(args.model_name_or_path, "optimizer.pt"))) scheduler.load_state_dict( torch.load(os.path.join(args.model_name_or_path, "scheduler.pt"))) if args.fp16: try: from apex import amp except ImportError: raise ImportError( "please install apex from https://www.github.com/nvidia/apex to use fp16 training." ) model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level) if args.n_gpu > 1: model = torch.nn.DataParallel(model) if args.local_rank != -1: model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True) global_step = 0 steps_trained_in_current_epoch = 0 # check for continuing training for a checkpoint if os.path.exists(args.model_name_or_path ) and "checkpoint" in args.model_name_or_path: global_step = int(args.model_name_or_path.split["-"][-1].split("/")[0]) epochs_trained = global_step // len( train_dataset) // args.gradient_accumulation_steps steps_trained_in_current_epoch = global_step % ( (len(train_dataloder)) // args.gradient_accumulation_steps) logger.info( "Continuing training from checkpoit, will skip to saved global_step" ) logger.info("Continuing training from epoch %d", epochs_trained) logger.info("Continuing trianing from global step %d", global_step) logger.info("will skip the first %d steps in the first epoch", steps_trained_in_current_epoch) tr_loss, logging_loss = 0.0, 0.0 model.zero_grad() seed_everyting(args.seed) for _ in range(int(args.num_train_epochs)): pbar = ProgressBar(n_total=len(train_dataloder), desc="Training") for step, batch in enumerate(train_dataloder): #skip past any already trained steps if resuming training if steps_trained_in_current_epoch > 0: steps_trained_in_current_epoch -= 1 continue model.train() batch = tuple(t.to(args.device) for t in batch) inputs = { "input_ids": batch[0], "attention_mask": batch[1], "labels": batch[3], "input_lens": batch[4] } if args.model_type != "distilbert": inputs["token_type_ids"] = (batch[2] if args.model_type in ["bert", "xlnet"] else None) outputs = model(**inputs) loss = outputs[0] if args.n_gpu > 1: loss = loss.mean( ) # mean to average on multi-gpu parallel trianing if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps if args.fp16: with amp.scale_loss(loss, optimizer) as scaled_loss: scaled_loss.backward() else: loss.backward() pbar(step, {"loss": loss.item()}) tr_loss += loss.item() if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16: torch.nn.utils.clip_grad_norm_( amp.master_params(optimizer), args.max_grad_norm) else: torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) scheduler.step() optimizer.step() model.zero_grad() global_step += 1 if args.local_rank in [ -1, 0 ] and args.logging_steps > 0 and global_step % args.logging_steps == 0: # Log metrics print(" ") if args.local_rank == -1: # Only evaluate when single GPU otherwise metrics may not average well evaluate(args, model, tokenizer) if args.local_rank in [ -1, 0 ] and args.save_steps > 0 and global_step % args.save_steps == 0: # Save model checkpoint output_dir = os.path.join( args.output_dir, "checkpoint-{}".format(global_step)) if not os.path.exists(output_dir): os.makedirs(output_dir) model_to_save = ( model.module if hasattr(model, "module") else model ) # Take care of distributed/parallel training model_to_save.save_pretrained(output_dir) torch.save(args, os.path.join(output_dir, "training_args.bin")) logger.info("Saving model checkpoint to %s", output_dir) tokenizer.save_vocabulary(output_dir) torch.save(optimizer.state_dict(), os.path.join(output_dir, "optimizer.pt")) torch.save(scheduler.state_dict(), os.path.join(output_dir, "scheduler.pt")) logger.info("Saving optimizer and scheduler states to %s", output_dir) logger.info("\n") if 'cuda' in str(args.device): torch.cuda.empty_cache() return global_step, tr_loss / global_step
def newmem(update, context): message = update.message chat = message.chat if message.from_user.id in get_chat_admins( context.bot, chat.id, context.bot_data.get("config").get("SUPER_ADMIN")): return for user in message.new_chat_members: if user.is_bot: continue num = SystemRandom().randint( 0, len(context.bot_data.get("config").get("CHALLENGE")) - 1) flag = context.bot_data.get("config").get("CHALLENGE")[num] if context.bot.restrict_chat_member( chat_id=chat.id, user_id=user.id, permissions=ChatPermissions(can_send_messages=False), ): logger.info( f"New member: Successfully restricted user {user.id} at group {chat.id}" ) else: logger.warning( f"New member: No enough permissions to restrict user {user.id} at group {chat.id}" ) buttons = [[ InlineKeyboardButton( flag.get("WRONG")[t], callback_data= f"challenge|{user.id}|{num}|{flag.get('wrong')[t]}", ) ] for t in range(len(flag.get("WRONG")))] buttons.append([ InlineKeyboardButton( flag.get("ANSWER"), callback_data=f"challenge|{user.id}|{num}|{flag.get('answer')}", ) ]) SystemRandom().shuffle(buttons) buttons.append([ InlineKeyboardButton( context.bot_data.get("config").get("PASS_BTN"), callback_data=f"admin|pass|{user.id}", ), InlineKeyboardButton( context.bot_data.get("config").get("KICK_BTN"), callback_data=f"admin|kick|{user.id}", ), ]) question_message = message.reply_text( context.bot_data.get("config").get("GREET").format( question=flag.get("QUESTION"), time=context.bot_data.get("config").get("TIME"), ), reply_markup=InlineKeyboardMarkup(buttons), parse_mode=ParseMode.MARKDOWN, ) context.job_queue.run_once( kick_queue, context.bot_data.get("config").get("TIME"), context={ "chat_id": chat.id, "user_id": user.id, }, name=f"{chat.id}|{user.id}|kick", ) context.job_queue.run_once( clean_queue, context.bot_data.get("config").get("TIME"), context={ "chat_id": chat.id, "user_id": user.id, "message_id": message.message_id, }, name=f"{chat.id}|{user.id}|clean_join", ) context.job_queue.run_once( clean_queue, context.bot_data.get("config").get("TIME"), context={ "chat_id": chat.id, "user_id": user.id, "message_id": question_message.message_id, }, name=f"{chat.id}|{user.id}|clean_question", )
def evaluate(args, model, tokenizer, prefix=""): metric = SeqEntityScore(args.id2label, markup=args.markup) eval_output_dir = args.output_dir if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]: os.makedirs(eval_output_dir) eval_dataset = load_and_cache_examples(args, args.task_name, tokenizer, data_type="dev") args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) # note distributedSampler samples randomly eval_sampler = SequentialSampler( eval_dataset) if args.local_rank == -1 else DistributedSampler( eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size, collate_fn=collate_fn) # Eval! logger.info("***** Running evaluation %s *****", prefix) logger.info(" Num examples = %d", len(eval_dataset)) logger.info(" Batch size = %d", args.eval_batch_size) eval_loss = 0.0 nb_eval_steps = 0 pbar = ProgressBar(n_total=len(eval_dataloader), desc="Evaluating") if isinstance(model, nn.DataParallel): model = model.module for step, batch in enumerate(eval_dataloader): model.eval() batch = tuple(t.to(args.device) for t in batch) with torch.no_grad(): inputs = { "input_ids": batch[0], "attention_mask": batch[1], "labels": batch[3], 'input_lens': batch[4] } if args.model_type != "distilbert": # XLM and RoBERTa don"t use segment_ids inputs["token_type_ids"] = (batch[2] if args.model_type in ["bert", "xlnet"] else None) outputs = model(**inputs) tmp_eval_loss, logits = outputs[:2] tags = model.crf.decode(logits, inputs['attention_mask']) if args.n_gpu > 1: tmp_eval_loss = tmp_eval_loss.mean( ) # mean() to average on multi-gpu parallel evaluating eval_loss += tmp_eval_loss.item() nb_eval_steps += 1 out_label_ids = inputs['labels'].cpu().numpy().tolist() input_lens = inputs['input_lens'].cpu().numpy().tolist() tags = tags.squeeze(0).cpu().numpy().tolist() if args.n_gpu > 1: tmp_eval_loss = tmp_eval_loss.mean( ) # mean() to average on multi-gpu parallel evaluating eval_loss += tmp_eval_loss.item() nb_eval_steps += 1 out_label_ids = inputs['labels'].cpu().numpy().tolist() input_lens = inputs['input_lens'].cpu().numpy().tolist() tags = tags.squeeze(0).cpu().numpy().tolist() for i, label in enumerate(out_label_ids): temp_1 = [] temp_2 = [] for j, m in enumerate(label): if j == 0: continue elif j == input_lens[i] - 1: metric.update(pred_paths=[temp_2], label_paths=[temp_1]) break else: temp_1.append(args.id2label[out_label_ids[i][j]]) temp_2.append(args.id2label[tags[i][j]]) pbar(step) logger.info("\n") eval_loss = eval_loss / nb_eval_steps eval_info, entity_info = metric.result() results = {f'{key}': value for key, value in eval_info.items()} results['loss'] = eval_loss logger.info("***** Eval results %s *****", prefix) info = "-".join( [f' {key}: {value:.4f} ' for key, value in results.items()]) logger.info(info) logger.info("***** Entity results %s *****", prefix) for key in sorted(entity_info.keys()): logger.info("******* %s results ********" % key) info = "-".join([ f' {key}: {value:.4f} ' for key, value in entity_info[key].items() ]) logger.info(info) return results
def log_info(logger, config): logger.info('--- Configs List---') logger.info('--- Dadaset:{}'.format(config.DATASET)) logger.info('--- Train:{}'.format(config.TRAIN)) logger.info('--- Bit:{}'.format(config.HASH_BIT)) logger.info('--- Alpha:{}'.format(config.alpha)) logger.info('--- Beta:{}'.format(config.beta)) logger.info('--- Lambda:{}'.format(config.lamb)) logger.info('--- Mu:{}'.format(config.mu)) logger.info('--- Batch:{}'.format(config.BATCH_SIZE)) logger.info('--- Lr_IMG:{}'.format(config.LR_IMG)) logger.info('--- Lr_TXT:{}'.format(config.LR_TXT))
def test_synchronisation_mongo_postgresql(verbose=True): """ Print synchronization status between Mongo and Postgres databases :param verbose: :return: """ output_str = " {0:25} {1:25} {2:>20} {3:>20}" synchronization = True client = pymongo.MongoClient(host=MONGO_HOST, port=MONGO_PORT) for postgres_url, mapping_name in zip(POSTGRES_URLS, MAPPINGS_NAMES): cur = psycopg2.connect(postgres_url).cursor() cur.execute( "SELECT table_name FROM information_schema.tables WHERE table_schema = 'synchro';" ) table_list = [t[0] for t in cur] with open('/home/data/' + mapping_name, 'r') as mapping_file: mapping = json.load(mapping_file) for mongo_database_name, db_mapping in mapping.items(): mongodb = client[mongo_database_name] for collection_name in mongodb.collection_names(): # Do not test synchronization for certain collections if collection_name not in db_mapping: if verbose: logger.warning( " collection '{}' present in MongoDB database '{}' is not mapped" .format(collection_name, mongo_database_name)) continue namespace = mongo_database_name + '.' + collection_name if 'namespaces' in CONFIG and not CONFIG['namespaces'].get( namespace, True): if verbose: logger.warning( " collection '{}' is ignored in doc_manager namespaces configuration." .format(collection_name)) continue # MongoDB count mongo_count = mongodb[collection_name].count() # PostgresSQL count table_name = to_sql_identifier(collection_name) query = "SELECT count(*) FROM {}".format(table_name) if table_name not in table_list: psql_count = 'not defined' else: cur.execute(query) psql_count = cur.fetchone()[0] # Test synchronization if mongo_count != psql_count: if synchronization: synchronization = False if verbose: logger.info( "\nPostgreSQL is not synchronized with MongoDB on the following collections:" ) logger.info( output_str.format("Database", "Collection", "MongoDB_count", "PostgreSQL_count")) if verbose: logger.info( output_str.format(mongo_database_name, collection_name, mongo_count, psql_count)) if synchronization and verbose: logger.info( "\nPostgreSQL is synchronized on MongoDB on dabasases and collection definded in the mapping." ) return synchronization