def description(self): schedule_description = read_file( os.path.join("bot", "templates", "schedule_description.tpl")) example_description = "<b>Commands:</b>\n" example_description += "--------\n".join(["{}\n".format(command.example_description)\ for command in schedule_commands]) return "{}\n{}".format(schedule_description, example_description)
def test_read_file_mixed_whitespace(mock_file_directory): LOG.info("test_read_file_from_mock_dir()") file_contents = "a ab abc abcd abcde" expected = file_contents.split() test_file = os.path.join(mock_file_directory, "abcde_whitespace.txt") actual = read_file(test_file) assert actual == expected
def parse_douban_captcha(): text = read_file(sys.argv[1]) root = etree.HTML(text) captcha_image = root.xpath('//img[@id="captcha_image"]/@src') captcha_id = root.xpath('//input[@name="captcha-id"]/@value') if captcha_image and captcha_id: print(captcha_image[0]) print(captcha_id[0])
def download_all_album_photos(root=DOUBAN_ROOT): id_file = os.path.join(root, 'album_ids.json') ids = read_file(id_file).split(',') print(len(ids)) blacklist = ['100945526', '100075813'] for id in ids: if id not in blacklist: executor.submit(download_album_photos, id, async_mode=False)
def get_longest_and_transposed_word_from_file(self, filepath): file_contents = read_file(filepath) if len(file_contents) == 0: raise Exception(f"{filepath} is empty\n") longest = get_longest_word(file_contents) reversed_word = reverse_string(longest) return (longest, reversed_word)
def joblist(self, euid): '''Getting the current joblist of euid as raw dict.''' # Sending one item dict with key `joblist` content = {'joblist': euid} write_file(self.runner_in, content, nodate=True, json_dumps=True) # Getting jobs: `{id: {}, id: {}, ...}` jobs = read_file(self.runner_out, whole=True, json_loads=True) return json.dumps(sorted(jobs.items()), indent=4) # Prettify-ing
def measure_scalebar(dir, old): for file in dir["files"]: image = read_file(file) if old: if "_contour" in file or "_mask" in file or "_crop" in file: continue length = measure_scalebar_old(image) else: length = measure_scalebar_new(image) append_or_change_filename(file, "Scale", None, "%s" % length)
def test_read_file(tmpdir): LOG.info("test_read_file()") file_contents = "a\nab\nabc\nabcd\nabcde" expected = file_contents.split() # create temp file for testing test_file = tmpdir.join("test_file.txt") test_file.write(file_contents) # read temp file actual = read_file(test_file) LOG.debug(actual) assert actual == expected
def run(src): subdirs = get_files_to_process(src) # print(subdirs) for dir in subdirs: for file in dir["files"]: image = read_file(file) backdrop = detect_backdrop(image) crop = crop_left_of_blue_line_hsv(image, backdrop=backdrop, visualize=True) filename = file.split("/")[-1] write_file(crop, "/Users/creimers/Downloads", filename)
def draw_black_box(target, old=False): outdir = get_target_dir(target["path"], "black-box", True) for file in target["files"]: image = read_file(file) backdrop = detect_backdrop(image) if backdrop == "white": image = trim_tape_edges(image) with_blue_line = crop_black_tape(image, backdrop, True) filename = file.split("/")[-1] outfile = os.path.join(outdir, filename) print(outfile) cv2.imwrite(outfile, with_blue_line)
def check_daemon_process(pid_file): '''Checks if the daemon process exists.''' if not os.path.isfile(pid_file): return False pid = read_file(pid_file, whole=True) proc = subprocess.Popen( shlex.split('/bin/ps -p {} -o cmd='.format(pid)), stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, ) ret = proc.wait() if not ret or b'hat/daemon_front.py' in proc.communicate()[0]: return True return False
class FilterDispatcherCommand(AbstractCommand): """/filter - returns a list of favorite filters""" example_description = read_file( os.path.join('bot', 'templates', 'examples', 'filter_example.tpl')) @login_required def handler(self, bot, update, *args, **kwargs): auth_data = kwargs.get('auth_data') options = kwargs.get('args') callback_data = 'filter_p:{}:{}' filter_buttons = list() filters = self.app.jira.get_favourite_filters(auth_data=auth_data) if options and filters: filter_name = ' '.join(options) if filter_name in filters.keys(): kwargs.update({ 'filter_name': filter_name, 'filter_id': filters.get(filter_name) }) return FilterIssuesCommand(self.app).handler( bot, update, *args, **kwargs) else: text = "Filter {} is not in your favorites".format(filter_name) return self.app.send(bot, update, text=text) elif filters: for name in filters.keys(): filter_buttons.append( InlineKeyboardButton(text=name, callback_data=callback_data.format( name, filters[name]))) buttons = InlineKeyboardMarkup(build_menu(filter_buttons, n_cols=2)) if buttons: text = "Pick up one of the filters:" return self.app.send(bot, update, text=text, buttons=buttons) self.app.send( bot, update, text= "You don't have any favourite filters. Add some filters in your Jira first" ) def command_callback(self): return CommandHandler('filter', self.handler, pass_args=True)
def draw_blue_line(target, old=False): outdir = get_target_dir(target["path"], "blue-line", True) for file in target["files"]: filename = file.split("/")[-1] outfile = os.path.join(outdir, filename) print(filename) try: image = read_file(file) backdrop = detect_backdrop(image) if backdrop == "white": image = trim_tape_edges(image) with_blue_line = crop_black_tape(image, backdrop, False) with_blue_line = crop_left_of_blue_line_hsv( with_blue_line, backdrop, old, True) # outfile = os.path.join(outdir, filename) # print(outfile) cv2.imwrite(outfile, with_blue_line) except Exception as e: print(e) click.secho(file, fg="red")
def binary_mask_parallel( dir, smoothen, old, clear, out_dir_name=None, no_black_tape=False ): """ create binary masks in parallel Args: dir <dict> : dictionary that holds the files to be processed index_threshold <float>: threshold for the mask by index function index_threshold <int>: threshold for the mask by threshold function smoothen <bool>: whether or not to smoothen the contour old <bool>: is this an "old" picture clear <bool>: should the output dir be cleared? out_dir_name <str>: alternative name for output dir no_black_tape <bool>: no black tape arround carrot """ method = "binary-masks" if out_dir_name is not None: dir_name = "__".join([method, out_dir_name]) else: dir_name = method target = get_target_dir(dir["path"], dir_name, clear) for file in dir["files"]: log_activity(file, method, False) try: image = read_file(file) filename = file.split("/")[-1] minimize = True binary_mask = create_binary_mask( image, smoothen=smoothen, minimize=minimize, old=old, no_black_tape=no_black_tape, ) write_file(binary_mask, target, filename) except Exception as error: click.secho(file, fg="red") click.secho(repr(error), fg="red")
def create_html_from_text(text_file, dst=None): if not isinstance(text_file, unicode): text_file = text_file.decode('utf-8') output = dst or os.path.dirname(text_file) # print('create_chapter from %s' % text_file) if not os.path.exists(output): os.makedirs(output) filename = os.path.basename(text_file) name, ext = os.path.splitext(filename) html_file = os.path.join(output, '%s.html' % name) if os.path.exists(html_file): return html_file, name else: text_lines = read_list(text_file) text_lines = ['<p>%s</p>' % line for line in text_lines] # first line as title, h2 body_str = '\n'.join(text_lines) html_tpl = read_file(CHAPTER_TEMPLATE) html_str = html_tpl.format(name, name, body_str) write_file(html_file, html_str) print('create_chapter to %s' % html_file) return html_file, name
def main(): dry_run = data['dry-run'] dtds = get_dtds(data['dtd'], data['mozilla-central']) print('======== DTDs ========') print(json.dumps(dtds, sort_keys=True, indent=2)) s = read_file(data['xul'], data['mozilla-central']) print('======== INPUT ========') print(s) print('======== OUTPUT ========') (new_xul, messages) = collect_messages(s, data['prefix']) print(new_xul) if not dry_run: write_file(data['xul'], new_xul, data['mozilla-central']) print('======== L10N ========') print(json.dumps(messages, sort_keys=True, indent=2)) migration = build_migration(messages, dtds, data) print('======== MIGRATION ========') print(migration) recipe_path = "{}/{}".format(data['migration'], data['recipe']) if not dry_run: write_file(recipe_path, migration, data['mozilla-central']) ftl = build_ftl(messages, dtds, data) print('======== Fluent ========') print(ftl.encode("utf-8")) if not dry_run: write_file(data['ftl'], ftl.encode("utf-8"), data['mozilla-central'], append=True)
def mask_overlay_parallel( dir, smoothen, old, clear, out_dir_name=None, no_black_tape=False ): """ create minary mask overlays in parallel """ method = MASK_OVERLAYS_DIR if out_dir_name is not None: dir_name = "__".join([method, out_dir_name]) else: dir_name = method target = get_target_dir(dir["path"], dir_name, clear) for file in dir["files"]: try: log_activity(file, method) image = read_file(file) masked_overlay = create_mask_overlay( image, smoothen=smoothen, old=old, no_black_tape=no_black_tape ) filename = file.split("/")[-1] write_file(masked_overlay, target, filename) except: click.secho(file, fg="red")
def description(self): return read_file( os.path.join('bot', 'templates', 'listunresolved_description.tpl'))
class ListUnresolvedIssuesCommand(AbstractCommand): """ /listunresolved <target> [name] - shows users or projects unresolved issues """ example_description = read_file( os.path.join('bot', 'templates', 'examples', 'listunresolved_example.tpl')) targets = ('my', 'user', 'project') @property def description(self): return read_file( os.path.join('bot', 'templates', 'listunresolved_description.tpl')) @staticmethod def get_argparsers(): my = CommandArgumentParser(prog='my', add_help=False) my.add_argument('target', type=str, choices=['my'], nargs='?') user = CommandArgumentParser(prog="user", add_help=False) user.add_argument( 'target', type=str, choices=['user'], ) user.add_argument('username', type=str) project = CommandArgumentParser(prog="project", add_help=False) project.add_argument('target', type=str, choices=['project']) project.add_argument('project_key', type=str) return [my, user, project] @with_progress() @login_required def handler(self, bot, update, *args, **kwargs): auth_data = kwargs.get('auth_data') arguments = kwargs.get('args') options = self.resolve_arguments(arguments, auth_data, False) if options.target == 'my': return UserUnresolvedCommand(self.app).handler( bot, update, username=auth_data.username, *args, **kwargs) elif options.target == 'user' and options.username: return UserUnresolvedCommand(self.app).handler( bot, update, username=options.username, *args, **kwargs) elif options.target == 'project' and options.project_key: ProjectUnresolvedCommand(self.app).handler( bot, update, project=options.project_key, *args, **kwargs) def _check_jira(self, options, auth_data): if options.target == 'my': self.app.jira.is_user_on_host(username=auth_data.username, auth_data=auth_data) elif options.target == 'user': self.app.jira.is_user_on_host(username=options.username, auth_data=auth_data) elif options.target == 'project': self.app.jira.is_project_exists(project=options.project_key, auth_data=auth_data) else: pass def command_callback(self): return CommandHandler('listunresolved', self.handler, pass_args=True) def validate_context(self, context): if not context: raise ContextValidationError(self.description) target = context.pop(0) # validate command options if target == 'my': if context: raise ContextValidationError( "<i>my</i> doesn't accept any arguments.") elif target == 'user': raise ContextValidationError( "<i>USERNAME</i> is a required argument.") elif target == 'project': raise ContextValidationError("<i>KEY</i> is a required argument.") else: raise ContextValidationError(f"Argument {target} not allowed.")
class ListStatusIssuesCommand(AbstractCommand): """ /liststatus <target> [name] - shows users or projects issues by a selected status """ example_description = read_file( os.path.join('bot', 'templates', 'examples', 'liststatus_example.tpl')) @property def description(self): return read_file( os.path.join('bot', 'templates', 'liststatus_description.tpl')) @staticmethod def get_argparsers(): my = CommandArgumentParser(prog="my", add_help=False) my.add_argument('target', type=str, choices=['my'], nargs='?') my.add_argument('status', type=str, nargs="*", action=ConcatAction) user = CommandArgumentParser(prog="user", add_help=False) user.add_argument('target', type=str, choices=['user'], nargs='?') user.add_argument('username', type=str) user.add_argument('status', type=str, nargs="*", action=ConcatAction) project = CommandArgumentParser(prog="project", add_help=False) project.add_argument('target', type=str, choices=['project'], nargs='?') project.add_argument('project_key', type=str) project.add_argument('status', type=str, nargs="*", action=ConcatAction) return [my, user, project] def _check_jira(self, options, auth_data): if options.status: self.app.jira.is_status_exists(status=options.status, auth_data=auth_data) if options.target == 'my': pass elif options.target == 'user': self.app.jira.is_user_on_host(username=options.username, auth_data=auth_data) elif options.target == 'project': self.app.jira.is_project_exists(project=options.project_key, auth_data=auth_data) @login_required def handler(self, bot, update, *args, **kwargs): auth_data = kwargs.get('auth_data') arguments = kwargs.get('args') options = self.resolve_arguments(arguments, auth_data) if options.target == 'my': if options.status: kwargs.update({ 'username': auth_data.username, 'status': options.status }) UserStatusIssuesCommand(self.app).handler( bot, update, *args, **kwargs) else: UserStatusIssuesMenu(self.app).handler( bot, update, username=auth_data.username, *args, **kwargs) elif options.target == 'user' and options.username: if options.status: kwargs.update({ 'username': options.username, 'status': options.status }) UserStatusIssuesCommand(self.app).handler( bot, update, *args, **kwargs) else: UserStatusIssuesMenu(self.app).handler( bot, update, username=options.username, *args, **kwargs) elif options.target == 'project' and options.project_key: if options.status: kwargs.update({ 'project': options.project_key, 'status': options.status }) ProjectStatusIssuesCommand(self.app).handler( bot, update, *args, **kwargs) else: ProjectStatusIssuesMenu(self.app).handler( bot, update, project=options.project_key, *args, **kwargs) def command_callback(self): return CommandHandler('liststatus', self.handler, pass_args=True) def validate_context(self, context): if not context: raise ContextValidationError(self.description) target = context.pop(0) if target == 'my': raise ContextValidationError( "<i>{status}</i> is a required argument.") elif target == 'user': raise ContextValidationError( "<i>{username}</i> and <i>{status}</i> are required arguments." ) elif target == 'project': raise ContextValidationError( "<i>{project_key}</i> and <i>{status}</i> are required arguments." ) else: raise ContextValidationError(f"Argument {target} not allowed.")
def run(args=None): card = CardInfo(hex_to_bytes(read_file(args.file))) print(card.get_timestamp())
def description(self): return read_file( os.path.join('bot', 'templates', 'start_description.tpl'))
def receive_from_daemon(self): return read_file( DAEMON_OUT, whole=True, json_loads=True )
'bug_id': '1411012', 'description': 'Migrate several strings from Preferences:Privacy', 'mozilla-central': '../mozilla-unified', 'xul': 'browser/components/preferences/in-content/{0}.xul'.format(pane), 'dtd': [ 'browser/locales/en-US/chrome/browser/preferences/{0}.dtd'.format(pane) ], 'migration': './migration.py', 'ftl': 'browser/locales/en-US/browser/preferences/{0}.ftl'.format(pane), } if __name__ == '__main__': dtds = get_dtds(data['dtd'], data['mozilla-central']) s = read_file(data['xul'], data['mozilla-central']) print('======== INPUT ========') print(s) print('======== OUTPUT ========') (new_xul, messages) = collect_messages(s) print(new_xul) write_file(data['xul'], new_xul, data['mozilla-central']) print('======== L10N ========') print(json.dumps(messages, sort_keys=True, indent=2)) migration = build_migration(messages, dtds, data)
class TimeTrackingCommand(AbstractCommand): """ /time <target> <name> [start_date] [end_date] - Shows spent time for users, issues and projects """ example_description = utils.read_file( os.path.join('bot', 'templates', 'examples', 'time_example.tpl')) targets = ('user', 'issue', 'project') available_days = ('today', 'yesterday') @property def description(self): return utils.read_file( os.path.join('bot', 'templates', 'time_description.tpl')) @staticmethod def get_argparsers(): issue = CommandArgumentParser(prog='issue', add_help=False) issue.add_argument('target', type=str, choices=['issue']) issue.add_argument('issue_key', type=str) issue.add_argument('start_date', type=str) issue.add_argument('end_date', type=str, nargs='?') user = CommandArgumentParser(prog='user', add_help=False) user.add_argument('target', type=str, choices=['user']) user.add_argument('username', type=str) user.add_argument('start_date', type=str) user.add_argument('end_date', type=str, nargs='?') project = CommandArgumentParser(prog='project', add_help=False) project.add_argument('target', type=str, choices=['project']) project.add_argument('project_key', type=str) project.add_argument('start_date', type=str) project.add_argument('end_date', type=str, nargs='?') return [issue, user, project] def _check_jira(self, options, auth_data): if options.target == 'issue': self.app.jira.is_issue_exists(issue=options.issue_key, auth_data=auth_data) elif options.target == 'user': self.app.jira.is_user_on_host(username=options.username, auth_data=auth_data) elif options.target == 'project': self.app.jira.is_project_exists(project=options.project_key, auth_data=auth_data) @login_required def handler(self, bot, update, *args, **kwargs): current_date = pendulum.now() arguments = kwargs.get('args') auth_data = kwargs.get('auth_data') options = self.resolve_arguments(arguments, auth_data, verbose=True) jira_timezone = self.app.jira.get_jira_tz(**kwargs) try: if options.start_date == 'today': date = self.__get_normalize_date(current_date.to_date_string(), jira_timezone) options.start_date = pendulum.create( date.year, date.month, date.day, tz=jira_timezone)._start_of_day() options.end_date = pendulum.create( date.year, date.month, date.day, tz=jira_timezone)._end_of_day() elif options.start_date == 'yesterday': date = self.__get_normalize_date( current_date.subtract(days=1).to_date_string(), self.app.jira.get_jira_tz(**kwargs)) options.start_date = pendulum.create( date.year, date.month, date.day, tz=jira_timezone)._start_of_day() options.end_date = pendulum.create( date.year, date.month, date.day, tz=jira_timezone)._end_of_day() else: if not options.end_date: start_date = self.__get_normalize_date( options.start_date, jira_timezone) end_date = self.__get_normalize_date( current_date.to_date_string(), jira_timezone) options.start_date = pendulum.create( start_date.year, start_date.month, start_date.day, tz=jira_timezone)._start_of_day() options.end_date = pendulum.create( end_date.year, end_date.month, end_date.day, tz=jira_timezone)._end_of_day() else: start_date = self.__get_normalize_date( options.start_date, jira_timezone) end_date = self.__get_normalize_date( options.end_date, jira_timezone) options.start_date = pendulum.create( start_date.year, start_date.month, start_date.day, tz=jira_timezone)._start_of_day() options.end_date = pendulum.create( end_date.year, end_date.month, end_date.day, tz=jira_timezone)._end_of_day() except ParserError: return self.app.send(bot, update, text='Invalid date format') kwargs['start_date'] = options.start_date kwargs['end_date'] = options.end_date if options.target == 'issue': kwargs['issue'] = options.issue_key return IssueTimeTrackerCommand(self.app).handler( bot, update, *args, **kwargs) elif options.target == 'user': kwargs['username'] = options.username return UserTimeTrackerCommand(self.app).handler( bot, update, *args, **kwargs) elif options.target == 'project': kwargs['project_key'] = options.project_key return ProjectTimeTrackerCommand(self.app).handler( bot, update, *args, **kwargs) def command_callback(self): return CommandHandler('time', self.handler, pass_args=True) def validate_context(self, context): if not context: raise ContextValidationError(self.description) target = context.pop(0) # validate command options if target == 'issue': raise ContextValidationError( "<i>{issue_key}</i> and <i>{start_date}</i> are required arguments." ) elif target == 'user': raise ContextValidationError( "<i>{username}</i> and <i>{start_date}</i> are required arguments." ) elif target == 'project': raise ContextValidationError( "<i>{project_key}</i> and <i>{start_date}</i> are required arguments." ) else: raise ContextValidationError(f"Argument {target} not allowed.") def __identify_of_date_format(self, date, timezone): """ The function determines and returns the date format based on the date and timezone :param date: date for parsing :param timezone: timezone for defining the format :return: date format for configuring library 'date_formats' dateparser """ from enum import Enum class NoValue(Enum): def __repr__(self): return '<%s.%s>' % (self.__class__.__name__, self.name) class DateFormatPatterns(NoValue): LITTLEENDIAN = r"\d{2}(.*?)\d{2}(.*?)\d{4}", MIDDLEENDIAN_0 = r"\w{3,}(.*?)\d{2}(.*?)\d{4}", MIDDLEENDIAN_1 = r"\d{2}(.*?)\w{3,}(.*?)\d{4}", BIGENDIAN_0 = r"\d{4}(.*?)\d{2}(.*?)\d{2}", BIGENDIAN_1 = r"\d{4}(.*?)\w{3,}(.*?)\d{2}", delimiters = "/.-" delimiter = str() for item in delimiters: if item in date: if delimiter and item != delimiter: raise DateTimeValidationError( f"Too many delimiters in date.") else: delimiter = item date_matches = { DateFormatPatterns.LITTLEENDIAN: "%m{dlm}%d{dlm}%Y".format(dlm=delimiter) if timezone in US_TIMEZONES else "%d{dlm}%m{dlm}%Y".format(dlm=delimiter), DateFormatPatterns.MIDDLEENDIAN_0: "%B{dlm}%d{dlm}%Y".format(dlm=delimiter), DateFormatPatterns.MIDDLEENDIAN_1: "%d{dlm}%B{dlm}%Y".format(dlm=delimiter), DateFormatPatterns.BIGENDIAN_0: "%Y{dlm}%m{dlm}%d".format(dlm=delimiter), DateFormatPatterns.BIGENDIAN_1: "%Y{dlm}%B{dlm}%d".format(dlm=delimiter), } for date_pattern in DateFormatPatterns: if re.match(re.compile(date_pattern.value[0]), date): return date_matches.get(date_pattern) def __get_normalize_date(self, date, timezone): try: date_fmt = self.__identify_of_date_format(date, timezone) return dateparser.parse(date, date_formats=[date_fmt], languages=['en', 'ru']) except TypeError: raise DateParsingError("Invalid format date.")
def run(args=None): logging.info("Starting AMC application with %s threads." % str(args.threads)) for combo in chunkify(read_file(args.input), args.threads): AMC(combo).start()
def description(self): return utils.read_file( os.path.join('bot', 'templates', 'time_description.tpl'))
def test_read_file_restricted_file(mock_file_directory): LOG.info("test_read_file_restricted_file()") file = os.path.join(mock_file_directory, "restricted.txt") with pytest.raises(PermissionError) as e: read_file(file)
def test_read_file_bad_filepath(mock_file_directory): LOG.info("test_read_file_bad_filepath()") file = os.path.join(mock_file_directory, "badfilepath.txt") with pytest.raises(FileNotFoundError) as e: read_file(file) LOG.debug(e)