def get_xfrs_token(page_html): """ Method to parse a D2L page to find the XSRF.Token. The token is returned as a string :param page_html: :return: """ soup = BeautifulSoup(page_html, "html.parser") # TODO Loop over all of them, as the location might change xsrf = str(soup.findAll("script")[0]).splitlines() token = None for line in xsrf: if "XSRF.Token" in line: # line_soup = re.findall("'(.*?)'", line) # We can also find our User.ID in this line as well for i in range(0, len(line_soup)): if line_soup[i] == 'XSRF.Token': token = line_soup[i + 1] break if token is None: logger.critical("Cannot find XSRF.Token. Code might have changed") exit(1) logger.debug("Found XSRF.Token. It's {}".format(token)) return token
def download(rqs, furl, path, level=3): if furl[0:4] != 'http': furl = "{}/{}".format(D2L_BASEURL, furl) file = rqs.get(furl, stream=True, headers={ "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0" }) if file.status_code == 302: # D2L, you don't f*****g redirect a 404/403 error. logger.error("Requested file is Not Found or Forbidden") if not os.path.isdir(safeFilePath(path)): # logger.info("Directory does not exist.") logger.debug(safeFilePath(path)) mkdir_recursive(safeFilePath(path)) try: name = furl.split('?')[0].split('/')[-1] if name == "DirectFileTopicDownload": name = file.headers['Content-Disposition'].split(';')[-1].split('=')[-1][1:-1] path += "/" + safeFilePath(name) with open(unquote(path), 'wb') as f: for chunk in tqdm.tqdm(file.iter_content(chunk_size=1024), desc="Downloading {}".format(name), position=level, unit="kb"): # is it kb or b? if chunk: # filter out keep-alive new chunks f.write(chunk) f.flush() except Exception as e: logger.exception("Exception caught during file download. {}", str(e))
def unlock_reruns(self): """Allow modification of max_reruns property""" logger.debug('Releasing rerun lock') self._reruns_locked = False
def on_task_input(self, task, config): # Create entries by parsing AniDB wishlist page html using beautifulsoup logger.verbose('Retrieving AniDB list: mywishlist:{}', config['mode']) task_headers = task.requests.headers.copy() task_headers['User-Agent'] = self.default_user_agent try: page = task.requests.get( self.anidb_url + 'animedb.pl', params=self._build_url_params(config), headers=task_headers, ) except RequestException as e: raise plugin.PluginError(str(e)) if page.status_code != 200: raise plugin.PluginError( 'Unable to get AniDB list. Either the list is private or does not exist.' ) entry_type = '' if config['type'] == 'movies': entry_type = 'Type: Movie' elif config['type'] == 'shows': entry_type = 'Type: TV Series' elif config['type'] == 'ovas': entry_type = 'Type: OVA' while True: soup = get_soup(page.text) soup_table = soup.find('table', class_='wishlist').find('tbody') trs = soup_table.find_all('tr') if not trs: logger.verbose( 'No movies were found in AniDB list: mywishlist') return for tr in trs: if tr.find('span', title=entry_type): a = tr.find('td', class_='name').find('a') if not a: logger.debug( 'No title link found for the row, skipping') continue anime_title = a.string if config.get('strip_dates'): # Remove year from end of series name if present anime_title = re.sub(r'\s+\(\d{4}\)$', '', anime_title) entry = Entry() entry['title'] = anime_title entry['url'] = self.anidb_url + a.get('href') entry['anidb_id'] = tr['id'][ 1:] # The <tr> tag's id is "aN..." where "N..." is the anime id logger.debug('{} id is {}', entry['title'], entry['anidb_id']) entry['anidb_name'] = entry['title'] yield entry else: logger.verbose('Entry does not match the requested type') try: # Try to get the link to the next page. next_link = soup.find('li', class_='next').find('a')['href'] except TypeError: # If it isn't there, there are no more pages to be crawled. logger.verbose('No more pages on the wishlist.') break comp_link = self.anidb_url + next_link logger.debug('Requesting: {}', comp_link) try: page = task.requests.get(comp_link, headers=task_headers) except RequestException as e: logger.error(str(e)) if page.status_code != 200: logger.warning('Unable to retrieve next page of wishlist.') break
def test_remove_simple(writer): i = logger.add(writer, format="{message}") logger.debug("1") logger.remove(i) logger.debug("2") assert writer.read() == "1\n"
def disconnect(self): logger.debug("Disconnecting") if self.ble_device is not None: self.ble_device.disconnect() self.ble_device = None self.__pin_already_sent = False
def username(self) -> str: username = self.specificity_credentials.get("USERNAME") if not username: username = self.credentials["USERNAME"] logger.debug("未能找到 {} 对应的账号,使用通用账号。", self.AUTH_NAME) return username
def main(args=None): """ Runs the .nfo generation. Use -h/--help to see all options. :param args: the command-line arguments to use, uses sys.argv if None :type args: list """ parser = argparse.ArgumentParser( description= 'Generates Kodi .nfo files with information retrieved from IMDB using local files with the unique IMDB movie ID.', formatter_class=argparse.ArgumentDefaultsHelpFormatter, prog="kodi-nfo-gen") parser.add_argument("--dir", metavar="DIR", dest="dir", required=True, help="the directory to traverse") parser.add_argument( "--type", dest="type", choices=["imdb"], default="imdb", required=False, help= "what type of ID the movie ID files represent, ie the website they are from" ) parser.add_argument("--recursive", action="store_true", dest="recursive", required=False, help="whether to traverse the directory recursively") parser.add_argument( "--pattern", metavar="GLOB", dest="pattern", required=False, default="*.imdb", help="the pattern for the files that contain the movie IDs") parser.add_argument( "--delay", metavar="SECONDS", dest="delay", type=int, required=False, default=1, help="the delay in seconds between web queries (to avoid blacklisting)" ) parser.add_argument( "--preferred_language", metavar="LANG", dest="language", required=False, default="en", help= "the preferred language for the titles (ISO 639-1, see https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)" ) parser.add_argument("--fanart", dest="fanart", choices=["none", "download", "use-existing"], default="none", required=False, help="how to deal with fan-art") parser.add_argument( "--fanart_file", metavar="FILE", dest="fanart_file", default="folder.jpg", required=False, help="when downloading or using existing fanart, use this filename") parser.add_argument( "--dry_run", action="store_true", dest="dry_run", required=False, help= "whether to perform a 'dry-run', ie only outputting the .nfo content to stdout but not saving it to files" ) parser.add_argument( "--overwrite", action="store_true", dest="overwrite", required=False, help= "whether to overwrite existing .nfo files, ie recreating them with freshly retrieved data" ) parser.add_argument("--verbose", action="store_true", dest="verbose", required=False, help="whether to output logging information") parser.add_argument("--debug", action="store_true", dest="debug", required=False, help="whether to output debugging information") parser.add_argument("--interactive", action="store_true", dest="interactive", required=False, help="for enabling interactive mode") parsed = parser.parse_args(args=args) # interactive mode turns on verbose mode if parsed.interactive and not (parsed.verbose or parsed.debug): parsed.verbose = True # configure loggin if parsed.debug: logging.basicConfig(level=logging.DEBUG) elif parsed.verbose: logging.basicConfig(level=logging.INFO) logger.debug(parsed) if parsed.interactive: logger.info("Entering interactive mode") generate(dir_=parsed.dir, idtype=parsed.type, recursive=parsed.recursive, pattern=parsed.pattern, dry_run=parsed.dry_run, overwrite=parsed.overwrite, language=parsed.language, fanart=parsed.fanart, fanart_file=parsed.fanart_file, interactive=parsed.interactive)
async def image_converter( current_user: models.Client = Depends(oauth2.get_current_user), reference_id: str = Form( ..., description="=6 character alphanumeric value"), company_name: str = Form(..., description="<30 character alphabetic value"), resize_width: Optional[int] = Form(None, description="image resize width"), resize_height: Optional[int] = Form(None, description="image resize height"), image_format: Optional[str] = Form( ..., description="any of [jpg, jpeg, png, tiff, tif, webp]"), quality_check: bool = Form(True, description="image quality check"), image_file: UploadFile = File(..., description="image file"), ): logger.debug({ "reference_id": reference_id, "company_name": company_name, "resize_width": resize_width, "resize_height": resize_height, "image_format": image_format, "quality_check": quality_check, }) process_start_time = time.time() try: # remove this to use marshmallow validation # input_data_dict = sch.ImageSchema().load( # {"reference_id": reference_id, # "resize_width": resize_width, # "resize_height": resize_height, # "company_name": company_name, # "quality_check": quality_check, # "image_format": image_format, # "image_file": image_file # }) # # input_data = utils.dict2obj(input_data_dict) # remove comment to use pydnatic validation image_path = await utils.save_file_aiof(image_file) input_data = sch.ImageSchema( reference_id=reference_id, resize_width=resize_width, resize_height=resize_height, company_name=company_name, quality_check=quality_check, image_format=image_format, image_file=image_path, ) except sch.ValidationError as e: raise utils.ValidationException(e) image = Image.fromarray(input_data.image_file) if resize_width is not None: width_percent = input_data.resize_width / float(image.size[0]) image_height = int(float(image.size[1]) * float(width_percent)) image = image.resize((input_data.resize_width, image_height), Image.ANTIALIAS) else: height_percent = input_data.resize_height / float(image.size[1]) image_width = int(float(image.size[0]) * float(height_percent)) image = image.resize((input_data.resize_height, image_width), Image.ANTIALIAS) buf = io.BytesIO() image.save(buf, format="JPEG") base64_string = base64.b64encode(buf.getvalue()).decode() time_stamp = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") data = { "base64_string": base64_string, "reference_id": input_data.reference_id, "time_stamp": time_stamp, "process_time": (time.time() - process_start_time), } return JSONResponse({"data": data, "status": "success"}, status_code=200)
from loguru import logger import pandas as pd import tcost_gui logger.debug('Transferring info from UI...') transfer = tcost_gui.main_dict logger.debug('Transferring info from UI..SUCCESS') logger.info(transfer) amount_of_receivers = set() cumul_of_rec = {} # получаем информацию о количестве активных групп доставки logger.debug('set of active delivery groups forming --->') for k, v in transfer.items(): amount_of_receivers.add(v[0]) logger.info(amount_of_receivers) # формируем словарь, где ключ - номер группы доставки logger.debug('dict of active delivery groups as keys forming --->') for i in amount_of_receivers: cumul_of_rec[str(i)] = 0 logger.info(cumul_of_rec) # для каждой группы доставки суммируем кол-во доставляемых ящиков for i in amount_of_receivers: for k, v in transfer.items(): if v[0] == i: cumul_of_rec[str(i)] += v[2] result_of_module = {}
def wrapper(*args, **kwargs): logger.debug(f'Entering {f}') result = f(*args, **kwargs) logger.debug(f'Exiting {f}') return result
def __enter__(self): bootstrap() logger.debug('Start recording invocations')
def __init__(self): logger.debug('creating instance of profiler') self.invocations = []
def shutdown(self, topic="", message=""): logger.info(f"Shutting down mqtt client {self.name}") self.client.loop_stop() logger.debug(f"Mqtt client {self.name} shut down")
def read_message(self): logger.debug("clearing the __new_message flag") self.__new_message = False
async def create_group_user( *, group: GroupUser, delay: int = Query( None, title=title, description=delay_description, ge=1, le=121, alias="delay", ), ) -> dict: """[summary] Add a user to a group Args: group (GroupUser): [description] delay (int, optional): [description]. Defaults to Query( None, title=title, description="Seconds to delay (max 121)", ge=1, \ le=121, alias="delay", ). Returns: dict: [description] Confirmation of user being added """ # sleep if delay option is used if delay is not None: logger.info(f"adding a delay of {delay} seconds") await asyncio.sleep(delay) check_id = str(group.group_id) group_id_exists = await check_id_exists(id=check_id) if group_id_exists is False: error: dict = {"error": f"Group ID '{check_id}' does not exist"} logger.warning(error) return JSONResponse(status_code=404, content=error) check_user = str(group.user) exist_user = await check_user_exists(user=check_user, group_id=check_id) if exist_user is True: error: dict = {"error": f"User ID '{check_id}' already in group"} logger.warning(error) return JSONResponse(status_code=400, content=error) try: user_id = str(uuid.uuid4()) group_data = { "id": user_id, "user": group.user, "group_id": group.group_id } logger.debug(group_data) # create group query = groups_item.insert() group_result = await execute_one_db(query=query, values=group_data) logger.debug(str(group_result)) full_result: dict = group_data logger.debug(full_result) return JSONResponse(status_code=status.HTTP_201_CREATED, content=full_result) except Exception as e: error: dict = {"error": str(e)} logger.debug(e) logger.critical(f"Critical Error: {e}") return JSONResponse(status_code=400, content=error)
def remove(self, downloads: List[Download], force: bool = False, files: bool = False, clean: bool = True) -> List[bool]: """ Remove the given downloads from the list. Parameters: downloads: the list of downloads to remove. force: whether to force the removal or not. files: whether to remove downloads files as well. clean: whether to remove the aria2 control file as well. Returns: Success or failure of the operation for each given download. """ # TODO: batch/multicall candidate if force: remove_func = self.client.force_remove else: remove_func = self.client.remove result = [] for download in downloads: if download.is_complete or download.is_removed or download.has_failed: logger.debug(f"Try to remove download result {download.gid}") try: self.client.remove_download_result(download.gid) except ClientException as error: logger.exception(error) result.append(error) else: logger.success(f"Removed download result {download.gid}") result.append(True) else: logger.debug(f"Try to remove download {download.gid}") try: removed_gid = remove_func(download.gid) except ClientException as error: logger.exception(error) result.append(error) else: logger.success(f"Removed download {download.gid}") result.append(True) try: self.client.remove_download_result(download.gid) except ClientException as error2: logger.debug( f"Failed to remove download result {download.gid}") logger.opt(exception=True).trace(error2) if removed_gid != download.gid: logger.debug( f"Removed download GID#{removed_gid} is different than download GID#{download.gid}" ) try: self.client.remove_download_result(removed_gid) except ClientException as error2: logger.debug( f"Failed to remove download result {removed_gid}" ) logger.opt(exception=True).trace(error2) if clean: # FUTURE: use missing_ok parameter on Python 3.8 try: download.control_file_path.unlink() except FileNotFoundError: logger.debug( f"aria2 control file {download.control_file_path} was not found" ) else: logger.debug( f"Removed control file {download.control_file_path}") if files and result[-1]: self.remove_files([download], force=True) return result
def multiregion(event): """ shows information for a given user's workspace need to pass username in the event """ logger.debug("workspaces.info.multiregion() starting") configuration = event.get('configuration', {}) user_workspaces = [] for region in configuration.get('regions'): # need to get the directoryid for each region if not configuration.get('directorymap', {}).get(region): logger.error(f"Couldn't find directoryid for region '{region}' in workspaces.info.multiregion()") continue try: session = Session( region_name=region ) except Exception as ERROR: # pylint: disable=broad-except,invalid-name logger.error(f"Failed to instantiate session with region {region}: {ERROR}") return False try: client = session.client('workspaces') findworkspace = client.describe_workspaces( DirectoryId=configuration.get('directorymap', {}).get(region), UserName=event.get('username'), ) if findworkspace.get('Workspaces'): for workspace in findworkspace.get('Workspaces'): wsid = workspace.get('WorkspaceId') state = workspace.get('State') bundlename = get_bundle_name(workspace.get('BundleId')) user_workspaces.append(f"{wsid} in state {state} (Region: {region} Bundle: {bundlename})") #pylint: disable=line-too-long except Exception as e: # pylint: disable=broad-except,invalid-name logger.error(f"ERROR: {e}") return False # message the user with the results slackclient = WebClient(token=configuration.get('slacktoken')) logger.debug("Messaging user to advise...") if not user_workspaces: slackclient.chat_postEphemeral( channel=configuration.get('channel_id'), user=configuration.get('user_id'), text=f"No workspaces found for username {event.get('username')}" ) else: blocks = [ { "type": "section", "text": { "type": "mrkdwn", "text": f"Workspace information for user {event.get('username')}" } }, ] for workspace in user_workspaces: blocks.append({ "type": "section", "text": { "type": "mrkdwn", "text": f"• {workspace}" } }) logger.debug(slackclient.chat_postEphemeral( channel=configuration.get('channel_id'), user=configuration.get('user_id'), blocks=blocks, )) return True
def test_edit_existing_level(writer): logger.level("DEBUG", no=20, icon="!") fmt = "{level.no}, <level>{level.name}</level>, {level.icon}, {message}" logger.add(writer, format=fmt, colorize=False) logger.debug("a") assert writer.read() == "20, DEBUG, !, a\n"
def main(args): root_cfg = cfg root_cfg.merge_from_file(args.config) logger.info("Load experiment configuration at: %s" % args.config) # resolve config root_cfg = complete_path_wt_root_in_cfg(root_cfg, ROOT_PATH) root_cfg = root_cfg.test task, task_cfg = specify_task(root_cfg) task_cfg.freeze() window_name = task_cfg.exp_name # build model model = model_builder.build(task, task_cfg.model) # build pipeline pipeline = pipeline_builder.build(task, task_cfg.pipeline, model) dev = torch.device(args.device) pipeline.set_device(dev) init_box = None template = None if len(args.init_bbox) == 4: init_box = args.init_bbox vw = None resize_ratio = args.resize dump_only = args.dump_only # create video stream if args.video == "webcam": logger.info("Starting video stream...") vs = cv2.VideoCapture(0) vs.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc('M', 'J', 'P', 'G')) elif not osp.isfile(args.video): logger.info("Starting from video frame image files...") vs = ImageFileVideoStream(args.video, init_counter=args.start_index) else: logger.info("Starting from video file...") vs = cv2.VideoCapture(args.video) # create video writer to output video if args.output: if osp.isdir(args.output): vw = ImageFileVideoWriter(args.output) else: fourcc = cv2.VideoWriter_fourcc(*'MJPG') width, height = vs.get(3), vs.get(4) vw = cv2.VideoWriter( args.output, fourcc, 25, (int(width * resize_ratio), int(height * resize_ratio))) # loop over sequence while vs.isOpened(): key = 255 ret, frame = vs.read() logger.debug("frame: {}".format(ret)) if ret: if template is not None: time_a = time.time() rect_pred = pipeline.update(frame) logger.debug(rect_pred) show_frame = frame.copy() time_cost = time.time() - time_a bbox_pred = xywh2xyxy(rect_pred) bbox_pred = tuple(map(int, bbox_pred)) cv2.putText(show_frame, "track cost: {:.4f} s".format(time_cost), (128, 20), cv2.FONT_HERSHEY_COMPLEX, font_size, (0, 0, 255), font_width) cv2.rectangle(show_frame, bbox_pred[:2], bbox_pred[2:], (0, 255, 0)) if template is not None: show_frame[:128, :128] = template else: show_frame = frame show_frame = cv2.resize( show_frame, (int(show_frame.shape[1] * resize_ratio), int(show_frame.shape[0] * resize_ratio))) # resize if not dump_only: cv2.imshow(window_name, show_frame) if vw is not None: vw.write(show_frame) # catch key if if (init_box is None) or (vw is None): logger.debug("Press key s to select object.") key = cv2.waitKey(30) & 0xFF logger.debug("key: {}".format(key)) if key == ord("q"): break # if the 's' key is selected, we are going to "select" a bounding # box to track elif key == ord("s"): # select the bounding box of the object we want to track (make # sure you press ENTER or SPACE after selecting the ROI) logger.debug("Select object to track") box = cv2.selectROI(window_name, frame, fromCenter=False, showCrosshair=True) if box[2] > 0 and box[3] > 0: init_box = box elif key == ord("c"): logger.debug( "init_box/template released, press key s to select object.") init_box = None template = None if (init_box is not None) and (template is None): template = cv2.resize( frame[int(init_box[1]):int(init_box[1] + init_box[3]), int(init_box[0]):int(init_box[0] + init_box[2])], (128, 128)) pipeline.init(frame, init_box) logger.debug( "pipeline initialized with bbox : {}".format(init_box)) vs.release() if vw is not None: vw.release() cv2.destroyAllWindows()
def cancel(self): """Send exit signal to thread""" logger.debug("Delayed call got cancellation signal") self._exit_early = True
def _add_proxy_url(self): urls = [] with open("proxy_url.txt", "r") as f: for line in f.readlines(): urls.append(line[:-1]) logger.info(f"number of proxy url {len(urls)}") for url in urls: logger.debug(f"update {url}") try: if len(self.ip_q["https"]) > 10: proxies = { "https": self._get_good_proxy("https", change_priority=False) } else: proxies = None resp = requests.get(f"https://{url}", timeout=15, proxies=proxies) regs = re.findall(r'((?:\d{1,3}\.){3}\d{1,3}):(\d+)', resp.text) for protocol in self.PROTOCOLS: for reg in regs: self.add_ip(protocol, reg[0] + ":" + reg[1]) logger.debug(f"done url {url}") except: try: if len(self.ip_q["http"]) > 10: proxies = { "http": self._get_good_proxy("http", change_priority=False) } else: proxies = None resp = requests.get(f"http://{url}", timeout=15, proxies=proxies) regs = re.findall(r'((?:\d{1,3}\.){3}\d{1,3}):(\d+)', resp.text) for protocol in self.PROTOCOLS: for reg in regs: self.add_ip(protocol, reg[0] + ":" + reg[1]) logger.debug(f"done url {url}") except: try: resp = requests.get(f"https://{url}", timeout=15) regs = re.findall(r'((?:\d{1,3}\.){3}\d{1,3}):(\d+)', resp.text) for protocol in self.PROTOCOLS: for reg in regs: self.add_ip(protocol, reg[0] + ":" + reg[1]) logger.debug(f"done url {url}") except: logger.info(f"bad url https://{url}") try: resp = requests.get(f"https://{url}", timeout=15) regs = re.findall( r'((?:\d{1,3}\.){3}\d{1,3}):(\d+)', resp.text) for protocol in self.PROTOCOLS: for reg in regs: self.add_ip(protocol, reg[0] + ":" + reg[1]) logger.debug(f"done url {url}") except: logger.info(f"bad url http://{url}")
def _post_install(self): logger.debug("Collecting tmproot files timestamps") tmproot_timestamps = self._collect_times() logger.debug("Dropping absolute paths from pkg-config") self._drop_absolute_pkgconfig_paths() logger.debug("Purging libtools' files") self._purge_libtools_files() # TODO: maybe this should be put into the configuration and not in orchestra itself logger.debug("Converting hardlinks to symbolic") self._hard_to_symbolic() # TODO: maybe this should be put into the configuration and not in orchestra itself logger.debug("Fixing RPATHs") self._fix_rpath() # TODO: this should be put into the configuration and not in orchestra itself logger.debug("Replacing NDEBUG preprocessor statements") self._replace_ndebug(self.build.ndebug) # TODO: this should be put into the configuration and not in orchestra itself logger.debug("Replacing ASAN preprocessor statements") self._replace_asan(self.build.asan) logger.debug("Restoring tmproot files timestamps") self._restore_mtimes(tmproot_timestamps) if self.build.component.license: logger.debug("Copying license file") source = self.build.component.license destination = installed_component_license_path(self.build.component.name, self.config) script = dedent( f""" DESTINATION_DIR="$TMP_ROOT$(dirname "{destination}")" mkdir -p "$DESTINATION_DIR" for DIR in "$BUILD_DIR" "$SOURCE_DIR"; do if test -e "$DIR/{source}"; then cp "$DIR/{source}" "$TMP_ROOT/{destination}" exit 0 fi done echo "Couldn't find {source}" exit 1 """ ) self._run_internal_script(script)
def send_pin(self): if not self.__pin_already_sent: logger.debug("Write PIN to {}", self.address) pin_handler = 0x24 self.ble_device.writeCharacteristic(pin_handler, self.pin, True) self.__pin_already_sent = True
def _save_hash_material(self): logger.debug("Saving hash material") hash_material_path = Path(self._hash_material_path()) hash_material_path.write_text(self.component.recursive_hash_material())
def test_remove_enqueue_filesink(tmpdir): file = tmpdir.join("test.log") i = logger.add(str(file), format="{message}", enqueue=True) logger.debug("1") logger.remove(i) assert file.read() == "1\n"
def debug(self, msg): logger.debug(msg)
def lock_reruns(self): """Prevent modification of max_reruns property""" logger.debug('Enabling rerun lock') self._reruns_locked = True
async def handle_task( websocket: WebSocket, db: ITaskflowDb = Depends(di.db), scheduler: TaskScheduler = Depends(di.scheduler), ): """ Websocket endpoint for starting a task """ task = None try: await websocket.accept() # The client sends the NewTask object data = await websocket.receive_json() new_task = NewTask.parse_obj(data) task_ = new_task.to_task() await db.insert_task(task_) task = task_ # Sends the resolved task await websocket.send_text(task.json()) # Waits for execution and sends updates periodically while True: pending_tasks_count = await db.get_pending_tasks_count() running_tasks_count = await db.get_running_tasks_count() message = SocketMessage( type=MessageType.INFO_UPDATE, data=ClientUpdateInfo( pending_tasks_count=pending_tasks_count, running_tasks_count=running_tasks_count, ), ) await websocket.send_text(message.json()) # Check if the task can be executed can_start = await scheduler.wait_for_task_execution(task) if can_start: message = SocketMessage(type=MessageType.TASK_CAN_START) await websocket.send_text(message.json()) task.is_running = True task.started_at = get_timestamp_ms() await db.update_task(task) break # Wait for task finish can_finish = False while not can_finish: data = await websocket.receive_json() try: message = SocketMessage.parse_obj(data) if message.type == MessageType.TASK_FINISH: can_finish = True elif message.type == MessageType.TASK_UPDATE: task = Task.parse_obj(message.data) logger.debug(task.json()) await db.update_task(task) except ValueError: logger.warning("Invalid JSON data") await websocket.close() except (WebSocketDisconnect, ConnectionClosed): logger.warning("Socket disconnected") except Exception as e: logger.exception("", e) await websocket.close() # Cleanup if task is not None: logger.info(f"Task {task.id} finished") await db.delete_task(task) del task
json_blob = json.loads(sanitized) # json_blob['Payload']['Html'] is literally rendered html that contains everything we need. But, it's HTML. # We're going to parse it with BS4 so we can read it. # Holy shit this worked page_soup = BeautifulSoup(json_blob['Payload']['Html'], "html.parser") # Now, we have a semi-nice table we can parse for all our courses. It's in the format of # | Course Name | Course Code | Semester | Standard Department | Start | End # and all we need to do is find all the `tr`- table row courses = page_soup.find_all('tr') parsed_courses = [] for course in courses: _course_soup = course.find_all('td') if len(_course_soup) < 5: logger.debug("Ignoring course row due to less than expected data length") continue metadata = {} # Find the name and ID. D2l puts the first column as a <th>, so we need to keep this mind name = course.find('th').find('a') metadata['name'] = name.text.strip().replace("/", "_") metadata['id'] = name.get('href').split('/')[-1] metadata['course_code'] = _course_soup[0].find('div').text.strip() metadata['semester'] = _course_soup[1].find('div').text.strip() metadata['dept'] = _course_soup[2].find('div').text.strip() metadata['start'] = _course_soup[3].find('div').text.strip() metadata['end'] = _course_soup[4].find('div').text.strip() parsed_courses.append(metadata)
def train_data(train_loader, val_loader, train_set, val_set, config, device): # define a loss function, and optimizer model = Classifier().to(device) criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=config['learning_rate']) num_epoch = config['num_epoch'] model_path = config['model_path'] # start training best_acc = 0.0 for epoch in range(num_epoch): train_acc = 0.0 train_loss = 0.0 val_acc = 0.0 val_loss = 0.0 # training model.train() # set the model to training mode for i, data in enumerate(train_loader): inputs, labels = data inputs, labels = inputs.to(device), labels.to(device) optimizer.zero_grad() outputs = model(inputs) batch_loss = criterion(outputs, labels) _, train_pred = torch.max(outputs, 1) # get the index of the class with the highest probability batch_loss.backward() optimizer.step() train_acc += (train_pred.cpu() == labels.cpu()).sum().item() train_loss += batch_loss.item() # validation if len(val_set) > 0: model.eval() # set the model to evaluation mode with torch.no_grad(): for i, data in enumerate(val_loader): inputs, labels = data inputs, labels = inputs.to(device), labels.to(device) outputs = model(inputs) batch_loss = criterion(outputs, labels) _, val_pred = torch.max(outputs, 1) val_acc += (val_pred.cpu() == labels.cpu()).sum().item() # get the index of the class with the highest probability val_loss += batch_loss.item() logger.debug('[{:03d}/{:03d}] Train Acc: {:3.6f} Loss: {:3.6f} | Val Acc: {:3.6f} loss: {:3.6f}'.format( epoch + 1, num_epoch, train_acc/len(train_set), train_loss/len(train_loader), val_acc/len(val_set), val_loss/len(val_loader) )) # use tensorboard to visualize the accuracy and loss real_train_acc = train_acc/len(train_set) real_train_loss = train_loss/len(train_loader) real_val_acc = val_acc/len(val_set) real_val_loss = val_loss/len(val_loader) writer.add_scalar("Acc/Val", real_val_acc, epoch) writer.add_scalar("Loss/Val", real_val_loss, epoch) writer.add_scalar("Acc/train", real_train_acc, epoch) writer.add_scalar("Loss/train", real_train_loss, epoch) # if the model improves, save a checkpoint at this epoch if val_acc > best_acc: best_acc = val_acc torch.save(model.state_dict(), model_path) logger.debug('saving model with acc {:.3f}'.format(best_acc/len(val_set))) else: logger.debug('[{:03d}/{:03d}] Train Acc: {:3.6f} Loss: {:3.6f}'.format( epoch + 1, num_epoch, train_acc/len(train_set), train_loss/len(train_loader) )) writer.flush() writer.close() # if not validating, save the last epoch if len(val_set) == 0: torch.save(model.state_dict(), model_path) logger.debug('saving model at last epoch')
def main(): # get device logger.debug('------------------------------------------------------------------------------------------------------') device = get_device() logger.debug(f'DEVICE: {device}') same_seeds(0) batch_size = config['batch_size'] val_ratio = config['val_ratio'] logger.debug('New train begins') logger.debug('Loading data ...') train_loader, val_loader, test_loader, test_set, train_set, val_set = prep_dataloader(batch_size, val_ratio) logger.debug('Loading data complete') train_data(train_loader, val_loader, train_set, val_set, config, device) predict_data(test_loader, device) logger.debug('All done')
async def group_id( group_id: str = Query( None, title="Group ID", description="Get by the Group UUID", alias="groupId", ), group_name: str = Query( None, title="Group Name", description="Get by the Group Name", alias="groupName", ), delay: int = Query( None, title=title, description=delay_description, ge=1, le=121, alias="delay", ), ) -> dict: """[summary] Get individual group data, including users Args: group_id (str, optional): [description]. Defaults to Query( None, title="Group ID", description="Get by the Group UUID", alias="groupId", ). group_name (str, optional): [description]. Defaults to Query( None, title="Group Name", description="Get by the Group Name", alias="groupName", ). delay (int, optional): [description]. Defaults to Query( None, title=title, description="Seconds to delay (max 121)", ge=1, le=121, \ alias="delay", ). Returns: dict: [description] Group data and associated users """ # sleep if delay option is used if delay is not None: await asyncio.sleep(delay) # if search by ID if group_id is not None: id_exists = await check_id_exists(group_id) if id_exists is False: error: dict = {"error": f"Group ID: '{group_id}' not found"} logger.warning(error) return JSONResponse(status_code=404, content=error) # elif search by name elif group_name is not None: name_exists = await check_unique_name(group_name) if name_exists is True: error: dict = {"error": f"Group Name: '{group_name}' not found"} logger.warning(error) return JSONResponse(status_code=404, content=error) query = groups.select().where(groups.c.name == group_name) name_result = await fetch_one_db(query=query) group_id = name_result["id"] # else at least one needs to be selected else: error: dict = {"error": "groupId or groupName must be used"} logger.warning(error) return JSONResponse(status_code=404, content=error) query = groups_item.select().where(groups_item.c.group_id == group_id) db_result = await fetch_all_db(query=query) users_list: list = [] user_dict: list = [] for r in db_result: logger.debug(r) user_data: dict = { "id": r["id"], "user": r["user"], "date_created": str(r["date_create"]), } user_dict.append(user_data) users_list.append(r["user"]) result = { "group_id": group_id, "count": len(users_list), "users": users_list, "user_info": user_dict, } return JSONResponse(status_code=200, content=result)