def test_start_subshell(self, call_mock, tempfile_mock): memfile = StringIO() memfile.name = 'FILENAME' tempfile_mock.return_value = memfile credentials = {'AWS_VALID_SECONDS': 600} start_subshell(credentials, 'ACCOUNT', 'ROLE') call_mock.assert_called_once_with( ["bash", "--rcfile", 'FILENAME'], stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin) expected = dedent(""" # Pretend to be an interactive, non-login shell for file in /etc/bash.bashrc ~/.bashrc; do [ -f "$file" ] && . "$file" done function afp_minutes_left { if ((SECONDS >= 600)) ; then echo EXPIRED else echo $(((600-SECONDS)/60)) Min fi } PS1="(AWS ACCOUNT/ROLE \$(afp_minutes_left)) $PS1" export AWS_VALID_SECONDS='600'""") memfile.seek(0) received = memfile.read() self.assertEqual(received, expected)
def test_alternative(self): mockfile = StringIO("I'm a file! *cough, cough*") mockfile.name = '/etc/mockfile.txt' m = encode(OrderedDict([ ('file', mockfile), ('full_name', 'Joe Average'), ])) boundary = m.get_boundary() self.assertEqual(str(m), '--{boundary}\r\n' 'Content-Disposition: form-data; name="file"; ' 'filename="mockfile.txt"\r\n' 'Content-Type: text/plain\r\n' '\r\n' "I'm a file! *cough, cough*\r\n" '--{boundary}\r\n' 'Content-Disposition: form-data; name="full_name"\r\n' 'Content-Type: text/plain; charset=utf-8\r\n' '\r\n' 'Joe Average\r\n' '--{boundary}--\r\n'.format( boundary=boundary ) )
def test_condition(self): pp = CompilerPreprocessor({"A": 1, "B": "2", "C": "0L"}) pp.out = StringIO() input = StringIO( dedent("""\ #ifdef A IFDEF_A #endif #if A IF_A #endif # if B IF_B # else IF_NOT_B # endif #if !C IF_NOT_C #else IF_C #endif """)) input.name = "foo" pp.do_include(input) self.assertEquals("IFDEF_A\nIF_A\nIF_NOT_B\nIF_NOT_C\n", pp.out.getvalue())
def test_condition(self): pp = CompilerPreprocessor({ 'A': 1, 'B': '2', 'C': '0L', }) pp.out = StringIO() input = StringIO(dedent('''\ #ifdef A IFDEF_A #endif #if A IF_A #endif # if B IF_B # else IF_NOT_B # endif #if !C IF_NOT_C #else IF_C #endif ''')) input.name = 'foo' pp.do_include(input) self.assertEquals('IFDEF_A\nIF_A\nIF_NOT_B\nIF_NOT_C\n', pp.out.getvalue())
def test_normalization(self): pp = CompilerPreprocessor({ '__has_attribute(bar)': '1', '__has_warning("-Wc++98-foo")': '1', }) pp.out = StringIO() input = StringIO( dedent('''\ #if __has_warning("-Wbar") WBAR #endif #if __has_warning("-Wc++98-foo") WFOO #endif #if !__has_warning("-Wc++98-foo") NO_WFOO #endif #if __has_attribute(bar) BAR #else NO_BAR #endif #if !__has_attribute(foo) NO_FOO #endif ''')) input.name = 'foo' pp.do_include(input) self.assertEquals(pp.out.getvalue(), 'WFOO\nBAR\nNO_FOO\n')
def test_normalization(self): pp = CompilerPreprocessor({ "__has_attribute(bar)": 1, '__has_warning("-Wc++98-foo")': 1 }) pp.out = StringIO() input = StringIO( dedent("""\ #if __has_warning("-Wbar") WBAR #endif #if __has_warning("-Wc++98-foo") WFOO #endif #if !__has_warning("-Wc++98-foo") NO_WFOO #endif #if __has_attribute(bar) BAR #else NO_BAR #endif #if !__has_attribute(foo) NO_FOO #endif """)) input.name = "foo" pp.do_include(input) self.assertEquals(pp.out.getvalue(), "WFOO\nBAR\nNO_FOO\n")
def test_start_subshell(self, call_mock, tempfile_mock): memfile = StringIO() memfile.name = 'FILENAME' tempfile_mock.return_value = memfile credentials = {'AWS_VALID_SECONDS': 600} start_subshell(credentials, 'ACCOUNT', 'ROLE') call_mock.assert_called_once_with(["bash", "--rcfile", 'FILENAME'], stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin) expected = dedent(""" # Pretend to be an interactive, non-login shell for file in /etc/bash.bashrc ~/.bashrc; do [ -f "$file" ] && . "$file" done function afp_minutes_left { if ((SECONDS >= 600)) ; then echo EXPIRED else echo $(((600-SECONDS)/60)) Min fi } PS1="(AWS ACCOUNT/ROLE \$(afp_minutes_left)) $PS1" export AWS_VALID_SECONDS='600'""") memfile.seek(0) received = memfile.read() self.assertEqual(received, expected)
def test_expansion(self): pp = CompilerPreprocessor({"A": 1, "B": "2", "C": "c", "D": "d"}) pp.out = StringIO() input = StringIO('A.B.C "D"') input.name = "foo" pp.do_include(input) self.assertEquals(pp.out.getvalue(), '1 . 2 . c "D"')
def test_expansion(self): pp = CompilerPreprocessor({'A': 1, 'B': '2', 'C': 'c', 'D': 'd'}) pp.out = StringIO() input = StringIO('A.B.C "D"') input.name = 'foo' pp.do_include(input) self.assertEquals(pp.out.getvalue(), '1 . 2 . c "D"')
def test_figure(self): # http://stackoverflow.com/a/2473445/1694979 imgfile = StringIO( 'GIF87a\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00ccc,' '\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;') imgfile.name = 'test_img_file.gif' self.q.figure.save('image', ContentFile(imgfile.read())) self.assertIsInstance(self.q.figure, ImageFieldFile)
def test_override_l10n(self): jm = self.jm jm.l10nbase = "/L10N_BASE" jm.outputFormat = "flat" # doesn't touch chrome dir without files jarcontents = StringIO("""en-US.jar: relativesrcdir dom/locales: """) jarcontents.name = "override.mn" jm.makeJar(jarcontents, "/NO_OUTPUT_REQUIRED") self.assertEquals(jm.localedirs, [os.path.join("/L10N_BASE", "dom")])
def test_override_l10n(self): jm = self.jm jm.l10nbase = '/L10N_BASE' jm.outputFormat = 'flat' # doesn't touch chrome dir without files jarcontents = StringIO('''en-US.jar: relativesrcdir dom/locales: ''') jarcontents.name = 'override.mn' jm.makeJar(jarcontents, '/NO_OUTPUT_REQUIRED') self.assertEquals(jm.localedirs, [os.path.join('/L10N_BASE', 'dom')])
def test_chunking(self): mockfile = StringIO("I'm a file! *cough, cough*") mockfile.name = 'mockfile.txt' ms = Multipart([ Field('full_name', fileobj=mockfile), Field('full_name', 'Joe Average') ], chunksize=10) self.assertTrue(reduce(lambda acc, x: acc and len(x) <= 10, ms, True))
def test_image_in_question(self): imgfile = StringIO( 'GIF87a\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00ccc,' '\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;') imgfile.name = 'test_img_file.gif' self.question1.figure.save('image', ContentFile(imgfile.read())) response = self.client.get('/tq1/take/') self.assertContains(response, '<img src=') self.assertContains(response, 'alt="' + str(self.question1.content))
def extract_templetor(fileobj, keywords, comment_tags, options): """Extract i18n messages from web.py templates.""" try: # Replace/remove inline js '\$' which interferes with the Babel python parser: code = web.template.Template.generate_code( fileobj.read().replace('\$', ''), fileobj.name) f = StringIO(code) f.name = fileobj.name except Exception as e: print(fileobj.name + ':', str(e), file=web.debug) return [] return extract_python(f, keywords, comment_tags, options)
def test_override(self): jm = self.jm jm.outputFormat = 'flat' # doesn't touch chrome dir without files jarcontents = StringIO('''en-US.jar: relativesrcdir dom/locales: ''') jarcontents.name = 'override.mn' jm.makeJar(jarcontents, '/NO_OUTPUT_REQUIRED') self.assertEquals(jm.localedirs, [ os.path.join(os.path.abspath('/TOPSOURCEDIR'), 'dom/locales', 'en-US') ])
def test_load_fasta(self): client = kbase_data.MockDataClient() client.authenticate() s = StringIO("> metadata\nGATTACAGATTACAGATTACA\n") s.name = "StringIO" src = kbase_data.FastaSource(s) oid = client.load(src) objects = client.search({'oid': oid}) #print("Got back objects:") #for o in objects: # print('--') # print(o) self.assertEqual(len(objects), 1) file_hash = objects[0].measurements.get('Bytes.FASTA', None) self.assertNotEqual(file_hash, None)
def test_override(self): jm = self.jm jm.outputFormat = "flat" # doesn't touch chrome dir without files jarcontents = StringIO("""en-US.jar: relativesrcdir dom/locales: """) jarcontents.name = "override.mn" jm.makeJar(jarcontents, "/NO_OUTPUT_REQUIRED") self.assertEquals( jm.localedirs, [ os.path.join(os.path.abspath("/TOPSOURCEDIR"), "dom/locales", "en-US") ], )
def test_start_subcmd(self, unlink_mock, call_mock, tempfile_mock): memfile = StringIO() memfile.name = 'FILENAME' # Need to mock away the 'close', so we can read it out later # despite 'close' being called inside the function. memfile.close = Mock() tempfile_mock.return_value = memfile credentials = {'AWS_VALID_SECONDS': 600} start_subcmd(credentials, 'ACCOUNT', 'ROLE') call_mock.assert_called_once_with(["cmd", "/K", 'FILENAME']) expected = dedent(""" @echo off set PROMPT=$C AWS ACCOUNT/ROLE $F set AWS_VALID_SECONDS='600'""") memfile.seek(0) received = memfile.read() self.assertEqual(received, expected)
def _expand_include_statements(file_obj, parsed_files=None): parsed_lines = [] for line in file_obj: line = line.strip() if not line or line.startswith('#'): continue match = re.match(SETTINGS_REGEX, line) if not match: parsed_lines.append(line) continue key = match.group(1).lower() value = match.group(2) if key != 'include': parsed_lines.append(line) continue if parsed_files is None: parsed_files = [] # The path can be relative to its parent configuration file if path.isabs(value) is False and value[0] != '~': folder = path.dirname(file_obj.name) value = path.join(folder, value) value = path.expanduser(value) for filename in glob.iglob(value): if path.isfile(filename): if filename in parsed_files: raise Exception( 'Include loop detected in ssh config file: %s' % filename, ) with open(filename) as fd: parsed_files.append(filename) parsed_lines.extend( _expand_include_statements(fd, parsed_files)) return parsed_lines output = StringIO('\n'.join(parsed_lines)) output.name = file_obj.name return output
def test_start_subcmd(self, unlink_mock, call_mock, tempfile_mock): memfile = StringIO() memfile.name = 'FILENAME' # Need to mock away the 'close', so we can read it out later # despite 'close' being called inside the function. memfile.close = Mock() tempfile_mock.return_value = memfile credentials = {'AWS_VALID_SECONDS': 600} start_subcmd(credentials, 'ACCOUNT', 'ROLE') call_mock.assert_called_once_with( ["cmd", "/K", 'FILENAME']) expected = dedent(""" @echo off set PROMPT=$C AWS ACCOUNT/ROLE $F set AWS_VALID_SECONDS='600'""") memfile.seek(0) received = memfile.read() self.assertEqual(received, expected)
def attempt_upload_file_and_verify_result(self, test_case, event_name, content=None): """ Running on a test case, creating a temp file, uploading it by calling the corresponding ajax event, and verifying that upload happens or is rejected as expected. """ if 'magic_number' in test_case: f_handler = StringIO(test_case['magic_number'].decode('hex')) elif content is not None: f_handler = StringIO(json.dumps(content, sort_keys=True)) else: f_handler = StringIO('') f_handler.content_type = test_case['mimetypes'] f_handler.name = 'file' + test_case['suffixes'] url = self.get_handler_url(event_name) resp = self.client.post(url, {'file': f_handler}) self.assertEqual(resp.status_code, test_case['status'])
def test_dive_export(mocked_urlopen, mock_fetch_response): mocked_urlopen.return_value = mock_fetch_response fragmentsdb = 'data/fragments.sqlite' uniprot_annot = StringIO( 'Entry\tGene names (primary )\tProtein families\tCross-reference (PDB)' + '\n' + 'P0CG48\tUBC\tUbiquitin family\t1C3T;2N2K' + '\n' ) pdbtag = StringIO('2n2k' + '\n') pdbtag.name = 'mytag' propnames = StringIO() props = StringIO() dive.dive_export(fragmentsdb, uniprot_annot, [pdbtag], propnames, props) assert '["pdb", "het", "fragment", "title", "smiles", "weight", "uniprot", "protein", "organism", "gene", "pdbtag", "family0", "family1", "family2", "family3", "family4"]' == propnames.getvalue() props_lines = props.getvalue().split('\n') result = list(filter(lambda d: d.startswith('2n2k_MTN_frag1'), props_lines))[0] expected = '2n2k_MTN_frag1 pdb:2n2k het:MTN fragment:1 "title:Ensemble structure of the closed state of Lys63-linked diubiquitin in the absence of a ligand" smiles:CC1(C)C=C(C[S-])C(C)(C)[NH+]1O 170.17 uniprot:P0CG48 "protein:Polyubiquitin-C" "organism:H**o sapiens" "gene:UBC" pdbtag:mytag "family0:Ubiquitin family"' assert result == expected
def test_alternative_explicit_filename(self): mockfile = StringIO("I'm a file! *cough, cough*") mockfile.name = '/etc/mockfile.txt' m = encode({ 'file': (mockfile, 'explicit-name.txt') }) boundary = m.get_boundary() self.assertEqual(str(m), '--{boundary}\r\n' 'Content-Disposition: form-data; name="file"; ' 'filename="explicit-name.txt"\r\n' 'Content-Type: text/plain\r\n' '\r\n' "I'm a file! *cough, cough*\r\n" '--{boundary}--\r\n'.format( boundary=boundary ) )
def test_with_fileobj(self): mockfile = StringIO("I'm a file! *cough, cough*") mockfile.name = 'mockfile.txt' m = Multipart([ Field('file', fileobj=mockfile) ], boundary='testing') self.assertEqual(m.get_headers(), { 'Content-Length': '150', 'Content-Type': 'multipart/form-data; boundary=testing' }) self.assertEqual(str(m), '--testing\r\n' 'Content-Disposition: form-data; name="file"; ' 'filename="mockfile.txt"\r\n' 'Content-Type: text/plain\r\n' '\r\n' "I'm a file! *cough, cough*\r\n" '--testing--\r\n' )
def test_dive_export(mocked_urlopen, mock_fetch_response): mocked_urlopen.return_value = mock_fetch_response fragmentsdb = 'data/fragments.sqlite' uniprot_annot = StringIO( 'Entry\tGene names (primary )\tProtein families\tCross-reference (PDB)' + '\n' + 'P0CG48\tUBC\tUbiquitin family\t1C3T;2N2K' + '\n') pdbtag = StringIO('2n2k' + '\n') pdbtag.name = 'mytag' propnames = StringIO() props = StringIO() dive.dive_export(fragmentsdb, uniprot_annot, [pdbtag], propnames, props) assert '["pdb", "het", "fragment", "title", "smiles", "weight", "uniprot", "protein", "organism", "gene", "pdbtag", "family0", "family1", "family2", "family3", "family4"]' == propnames.getvalue( ) props_lines = props.getvalue().split('\n') result = list(filter(lambda d: d.startswith('2n2k_MTN_frag1'), props_lines))[0] expected = '2n2k_MTN_frag1 pdb:2n2k het:MTN fragment:1 "title:Ensemble structure of the closed state of Lys63-linked diubiquitin in the absence of a ligand" smiles:CC1(C)C=C(C[S-])C(C)(C)[NH+]1O 170.17 uniprot:P0CG48 "protein:Polyubiquitin-C" "organism:H**o sapiens" "gene:UBC" pdbtag:mytag "family0:Ubiquitin family"' assert result == expected
def fetch(self, url, inject_ids=True): # type: (Text, bool) -> Any if url in self.idx: return self.idx[url] try: text = self.fetch_text(url) if isinstance(text, bytes): textIO = StringIO(text.decode('utf-8')) else: textIO = StringIO(text) textIO.name = url # type: ignore result = yaml.round_trip_load(textIO) add_lc_filename(result, url) except yaml.parser.ParserError as e: raise validate.ValidationException("Syntax error %s" % (e)) if (isinstance(result, CommentedMap) and inject_ids and bool(self.identifiers)): for identifier in self.identifiers: if identifier not in result: result[identifier] = url self.idx[self.expand_url(result[identifier], url)] = result else: self.idx[url] = result return result
def send_details_email(self, save=True, template_name=TemplateModel.MEETING, mail_to=None, exclude_user=None, extra_ctx=None, save_timestamp=True): ctx = { 'meeting': self.closest_repetition.to_meeting_with_repetition_date(), 'site': Site.objects.get_current(), 'protocol': settings.SSL_ON and 'https' or 'http', 'link': self.get_absolute_url(), } if extra_ctx: ctx.update(extra_ctx) if mail_to is None: mail_to = self.members for member in mail_to: if not member.user.is_active or not member.is_active \ or member.invitation_status != member.INV_INVITED: continue if exclude_user and member.user.id == exclude_user.id: continue ctx['recipient'] = member tmpl = TemplateModel.objects.get(name=template_name) subject = tmpl.generate_title( ctx) or self.account.name # fixme: which one? message = tmpl.generate(ctx) mail = EmailMessage(subject, message, settings.DEFAULT_FROM_EMAIL, [member.user.email]) mail.extra_headers['From'] = settings.DONOTREPLY_FROM_EMAIL admin_emails = [ m.user.email for m in self.account.get_admin_memberships() if m.id != member.id ] if self.created_by and self.created_by.user.email: mail.extra_headers['Reply-To'] = self.created_by.user.email elif admin_emails: mail.extra_headers['Reply-To'] = ', '.join(admin_emails) mail.content_subtype = "html" # try: # for doc in self.folder.documents.all(): # mail.attach(doc.name, doc.file.read()) # except Folder.DoesNotExist: # pass ics = self.generate_ics(self.creator.user, member) f = StringIO() f.name = 'calendar.ics' f.write(ics) mail.attach(f.name, f.getvalue(), 'text/calendar') mail.send() f.close() if save_timestamp: self.last_email_sent = timezone.now() if save: self.save()
def clipboard_stream(name=None): stream = StringIO(get_clipboard_data()) stream.name = name or '<clipboard>' return stream
def main(): parser = ArgumentParser() parser.add_argument( "-D", dest="defines", action="append", metavar="VAR[=VAL]", help="Define a variable", ) parser.add_argument( "--format", default="omni", help="Choose the chrome format for packaging " + "(omni, jar or flat ; default: %(default)s)", ) parser.add_argument("--removals", default=None, help="removed-files source file") parser.add_argument( "--ignore-errors", action="store_true", default=False, help="Transform errors into warnings.", ) parser.add_argument( "--ignore-broken-symlinks", action="store_true", default=False, help="Do not fail when processing broken symlinks.", ) parser.add_argument( "--minify", action="store_true", default=False, help="Make some files more compact while packaging", ) parser.add_argument( "--minify-js", action="store_true", help="Minify JavaScript files while packaging.", ) parser.add_argument( "--js-binary", help="Path to js binary. This is used to verify " "minified JavaScript. If this is not defined, " "minification verification will not be performed.", ) parser.add_argument("--jarlog", default="", help="File containing jar " + "access logs") parser.add_argument( "--compress", choices=("none", "deflate"), default="deflate", help="Use given jar compression (default: deflate)", ) parser.add_argument("manifest", default=None, nargs="?", help="Manifest file name") parser.add_argument("source", help="Source directory") parser.add_argument("destination", help="Destination directory") parser.add_argument( "--non-resource", nargs="+", metavar="PATTERN", default=[], help="Extra files not to be considered as resources", ) args = parser.parse_args() defines = dict(buildconfig.defines["ALLDEFINES"]) if args.ignore_errors: errors.ignore_errors() if args.defines: for name, value in [split_define(d) for d in args.defines]: defines[name] = value compress = { "none": False, "deflate": True, }[args.compress] copier = FileCopier() if args.format == "flat": formatter = FlatFormatter(copier) elif args.format == "jar": formatter = JarFormatter(copier, compress=compress) elif args.format == "omni": formatter = OmniJarFormatter( copier, buildconfig.substs["OMNIJAR_NAME"], compress=compress, non_resources=args.non_resource, ) else: errors.fatal("Unknown format: %s" % args.format) # Adjust defines according to the requested format. if isinstance(formatter, OmniJarFormatter): defines["MOZ_OMNIJAR"] = 1 elif "MOZ_OMNIJAR" in defines: del defines["MOZ_OMNIJAR"] respath = "" if "RESPATH" in defines: respath = SimpleManifestSink.normalize_path(defines["RESPATH"]) while respath.startswith("/"): respath = respath[1:] with errors.accumulate(): finder_args = dict( minify=args.minify, minify_js=args.minify_js, ignore_broken_symlinks=args.ignore_broken_symlinks, ) if args.js_binary: finder_args["minify_js_verify_command"] = [ args.js_binary, os.path.join(os.path.abspath(os.path.dirname(__file__)), "js-compare-ast.js"), ] finder = PackagerFileFinder(args.source, find_executables=True, **finder_args) if "NO_PKG_FILES" in os.environ: sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None) else: sinkformatter = formatter sink = SimpleManifestSink(finder, sinkformatter) if args.manifest: preprocess_manifest(sink, args.manifest, defines) else: sink.add(Component(""), "bin/*") sink.close(args.manifest is not None) if args.removals: removals_in = StringIO(open(args.removals).read()) removals_in.name = args.removals removals = RemovedFiles(copier) preprocess(removals_in, removals, defines) copier.add(mozpath.join(respath, "removed-files"), removals) # If a pdb file is present and we were instructed to copy it, include it. # Run on all OSes to capture MinGW builds if buildconfig.substs.get("MOZ_COPY_PDBS"): # We want to mutate the copier while we're iterating through it, so copy # the items to a list first. copier_items = [(p, f) for p, f in copier] for p, f in copier_items: if isinstance(f, ExecutableFile): pdbname = os.path.splitext(f.inputs()[0])[0] + ".pdb" if os.path.exists(pdbname): copier.add(os.path.basename(pdbname), File(pdbname)) # Setup preloading if args.jarlog: if not os.path.exists(args.jarlog): raise Exception("Cannot find jar log: %s" % args.jarlog) omnijars = [] if isinstance(formatter, OmniJarFormatter): omnijars = [ mozpath.join(base, buildconfig.substs["OMNIJAR_NAME"]) for base in sink.packager.get_bases(addons=False) ] from mozpack.mozjar import JarLog log = JarLog(args.jarlog) for p, f in copier: if not isinstance(f, Jarrer): continue if respath: p = mozpath.relpath(p, respath) if p in log: f.preload(log[p]) elif p in omnijars: raise Exception("No jar log data for %s" % p) copier.copy(args.destination) generate_precomplete( os.path.normpath(os.path.join(args.destination, respath)))
def main(): parser = ArgumentParser() parser.add_argument('-D', dest='defines', action='append', metavar="VAR[=VAL]", help='Define a variable') parser.add_argument('--format', default='omni', help='Choose the chrome format for packaging ' + '(omni, jar or flat ; default: %(default)s)') parser.add_argument('--removals', default=None, help='removed-files source file') parser.add_argument('--ignore-errors', action='store_true', default=False, help='Transform errors into warnings.') parser.add_argument('--ignore-broken-symlinks', action='store_true', default=False, help='Do not fail when processing broken symlinks.') parser.add_argument('--minify', action='store_true', default=False, help='Make some files more compact while packaging') parser.add_argument('--minify-js', action='store_true', help='Minify JavaScript files while packaging.') parser.add_argument('--js-binary', help='Path to js binary. This is used to verify ' 'minified JavaScript. If this is not defined, ' 'minification verification will not be performed.') parser.add_argument('--jarlog', default='', help='File containing jar ' + 'access logs') parser.add_argument('--compress', choices=('none', 'deflate', 'brotli'), default='deflate', help='Use given jar compression (default: deflate)') parser.add_argument('manifest', default=None, nargs='?', help='Manifest file name') parser.add_argument('source', help='Source directory') parser.add_argument('destination', help='Destination directory') parser.add_argument('--non-resource', nargs='+', metavar='PATTERN', default=[], help='Extra files not to be considered as resources') args = parser.parse_args() defines = dict(buildconfig.defines['ALLDEFINES']) if args.ignore_errors: errors.ignore_errors() if args.defines: for name, value in [split_define(d) for d in args.defines]: defines[name] = value compress = { 'none': False, 'deflate': True, 'brotli': JAR_BROTLI, }[args.compress] copier = FileCopier() if args.format == 'flat': formatter = FlatFormatter(copier) elif args.format == 'jar': formatter = JarFormatter(copier, compress=compress) elif args.format == 'omni': formatter = OmniJarFormatter(copier, buildconfig.substs['OMNIJAR_NAME'], compress=compress, non_resources=args.non_resource) else: errors.fatal('Unknown format: %s' % args.format) # Adjust defines according to the requested format. if isinstance(formatter, OmniJarFormatter): defines['MOZ_OMNIJAR'] = 1 elif 'MOZ_OMNIJAR' in defines: del defines['MOZ_OMNIJAR'] respath = '' if 'RESPATH' in defines: respath = SimpleManifestSink.normalize_path(defines['RESPATH']) while respath.startswith('/'): respath = respath[1:] if not buildconfig.substs['CROSS_COMPILE']: launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist') with errors.accumulate(): finder_args = dict( minify=args.minify, minify_js=args.minify_js, ignore_broken_symlinks=args.ignore_broken_symlinks, ) if args.js_binary: finder_args['minify_js_verify_command'] = [ args.js_binary, os.path.join(os.path.abspath(os.path.dirname(__file__)), 'js-compare-ast.js') ] finder = FileFinder(args.source, find_executables=True, **finder_args) if 'NO_PKG_FILES' in os.environ: sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None) else: sinkformatter = formatter sink = SimpleManifestSink(finder, sinkformatter) if args.manifest: preprocess_manifest(sink, args.manifest, defines) else: sink.add(Component(''), 'bin/*') sink.close(args.manifest is not None) if args.removals: removals_in = StringIO(open(args.removals).read()) removals_in.name = args.removals removals = RemovedFiles(copier) preprocess(removals_in, removals, defines) copier.add(mozpath.join(respath, 'removed-files'), removals) # shlibsign libraries if launcher.can_launch(): if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'): for lib in SIGN_LIBS: libbase = mozpath.join(respath, '%s%s') \ % (buildconfig.substs['DLL_PREFIX'], lib) libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX']) if copier.contains(libname): copier.add( libbase + '.chk', LibSignFile(os.path.join(args.destination, libname))) # If a pdb file is present and we were instructed to copy it, include it. # Run on all OSes to capture MinGW builds if buildconfig.substs.get('MOZ_COPY_PDBS'): # We want to mutate the copier while we're iterating through it, so copy # the items to a list first. copier_items = [(p, f) for p, f in copier] for p, f in copier_items: if isinstance(f, ExecutableFile): pdbname = os.path.splitext(f.inputs()[0])[0] + '.pdb' if os.path.exists(pdbname): copier.add(os.path.basename(pdbname), File(pdbname)) # Setup preloading if args.jarlog: if not os.path.exists(args.jarlog): raise Exception('Cannot find jar log: %s' % args.jarlog) omnijars = [] if isinstance(formatter, OmniJarFormatter): omnijars = [ mozpath.join(base, buildconfig.substs['OMNIJAR_NAME']) for base in sink.packager.get_bases(addons=False) ] from mozpack.mozjar import JarLog log = JarLog(args.jarlog) for p, f in copier: if not isinstance(f, Jarrer): continue if respath: p = mozpath.relpath(p, respath) if p in log: f.preload(log[p]) elif p in omnijars: raise Exception('No jar log data for %s' % p) copier.copy(args.destination) generate_precomplete( os.path.normpath(os.path.join(args.destination, respath)))
def parse_attachment(message_part, state, attachments=None): """ Extract the attachment and metadata about it from the message. Returns the content, content type, size, and create/modification/read dates for the attachment. """ params = message_part.get_params(None, "Content-Disposition") if params: # If a 'part' has a Content-Disposition, we assume it is an attachment try: params = dict(params) print("\tContent-Disposition (for following email)", params) if "attachment" in params: # Not sure what's going on here # Why get payload with decode, then try again and reparse? # See details at # http://docs.python.org/2/library/email.message.html#email.message.Message.get_payload file_data = message_part.get_payload(decode=True) if not file_data: payload = message_part.get_payload() if isinstance(payload, list): for msgobj in payload: # TODO not sure this actually does anything parse2(msgobj, state, attachments) return None print(message_part.get_payload(), file=sys.stderr) print(message_part.get_content_charset(), file=sys.stderr) attachment = StringIO(file_data) attachment.content_type = message_part.get_content_type() attachment.size = params.get("size", len(file_data)) attachment.create_date = params.get("create-date") attachment.mod_date = params.get("modification-date") attachment.read_date = params.get("read-date") # TODO convert dates to datetime filename = message_part.get_filename(None) if filename: # Filenames may be encoded with =?encoding?... # If so, convert to unicode name, encoding = email.header.decode_header(filename)[0] if encoding: print( "\t{filename} encoded with {encoding}, converting to unicode" .format(filename=filename, encoding=encoding)) filename = name.decode(encoding) else: # filename not in Content-Disposition print( """Warning, no filename found in: [{%s}%s] Content-Disposition: %s or Content-Type""" % (state.sourceFileUUID, state.sourceFilePath, params), file=sys.stderr, ) filename = six.text_type(uuid.uuid4()) print( "Attempting extraction with random filename: %s" % (filename), file=sys.stderr, ) # Remove newlines from filename because that breaks everything filename = filename.replace("\r", "").replace("\n", "") attachment.name = filename return attachment except Exception as inst: print(type(inst), file=sys.stderr) print(inst.args, file=sys.stderr) print( "Error parsing: file: {%s}%s" % (state.sourceFileUUID, state.sourceFilePath), file=sys.stderr, ) print("Error parsing: Content-Disposition: ", params, file=sys.stderr) print(file=sys.stderr) state.error_count += 1 return None