def dump(self):
posOrig = self.pos
self.printAndSet("Type", self.readuInt32())
self.printAndSet("Size", self.readuInt32(), hexdump=False)
wmfrecord.RectL(self, "Bounds").dump()
self.printAndSet("xDest", self.readInt32(), hexdump=False)
self.printAndSet("yDest", self.readInt32(), hexdump=False)
self.printAndSet("xSrc", self.readInt32(), hexdump=False)
self.printAndSet("ySrc", self.readInt32(), hexdump=False)
self.printAndSet("cxSrc", self.readInt32(), hexdump=False)
self.printAndSet("cySrc", self.readInt32(), hexdump=False)
self.printAndSet("offBmiSrc", self.readuInt32(), hexdump=False)
self.printAndSet("cbBmiSrc", self.readuInt32(), hexdump=False)
self.printAndSet("offBitsSrc", self.readuInt32(), hexdump=False)
self.printAndSet("cbBitsSrc", self.readuInt32(), hexdump=False)
self.printAndSet("UsageSrc", self.readInt32(), dict=DIBColors)
self.printAndSet("BitBltRasterOperation", self.readuInt32(), dict=wmfrecord.RasterPolishMap)
self.printAndSet("cxDest", self.readInt32(), hexdump=False)
self.printAndSet("cyDest", self.readInt32(), hexdump=False)
print '<BitmapBuffer>'
if self.cbBmiSrc:
self.pos = posOrig + self.offBmiSrc
self.BmiSrc = self.readBytes(self.cbBmiSrc)
print '<BmiSrc value="%s"/>' % base64.b64encode(self.BmiSrc)
if self.cbBitsSrc:
self.pos = posOrig + self.offBitsSrc
self.BitsSrc = self.readBytes(self.cbBitsSrc)
print '<BitsSrc value="%s"/>' % base64.b64encode(self.BitsSrc)
print '</BitmapBuffer>'
assert self.pos - posOrig == self.Size
def issue_certificate_request(self, order_id, order_meta, plugin_meta,
barbican_meta_dto):
if barbican_meta_dto.generated_csr is not None:
encoded_csr = barbican_meta_dto.generated_csr
else:
try:
encoded_csr = base64.b64decode(order_meta['request_data'])
except KeyError:
return cert_manager.ResultDTO(
cert_manager.CertificateStatus.CLIENT_DATA_ISSUE_SEEN,
status_message=u._("No request_data specified"))
csr = crypto.load_certificate_request(crypto.FILETYPE_PEM, encoded_csr)
ca_id = barbican_meta_dto.plugin_ca_id
if ca_id:
ca = self.cas.get(ca_id)
if ca is None:
raise cert_manager.CertificateGeneralException(
"Invalid ca_id passed into snake oil plugin:" + ca_id)
else:
ca = self.ca
cert_mgr = CertManager(ca)
cert = cert_mgr.make_certificate(csr)
cert_enc = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
return cert_manager.ResultDTO(
cert_manager.CertificateStatus.CERTIFICATE_GENERATED,
certificate=base64.b64encode(cert_enc),
intermediates=base64.b64encode(ca.pkcs7))
def request_authentication(self):
'''
Performs HTTP authentication request.
Returns user specific access token.
'''
# Initialization information
encodedUserInfo = base64.b64encode(config.clientID)
encodedPassword = base64.b64encode(config.password)
encodedSecurityInfo = base64.b64encode( \
encodedUserInfo + ':' + encodedPassword)
# Authentication request setup
data = {'grant_type': 'client_credentials'}
headers = { \
'content-type' : 'application/x-www-form-urlencoded', \
'Authorization': 'Basic ' + encodedSecurityInfo \
}
# Request authentication
data = urllib.urlencode(data)
request = urllib2.Request(config.auth_url, data, headers)
response = json.loads(urllib2.urlopen(request).read())
# Return access token
return response['access_token']
def serial_elgamal_msg(cryptsymkey):
"""Get a 2-elements tuple of str(), return a string."""
try:
ns = b64encode(cryptsymkey[0]) + ':' + \
b64encode(cryptsymkey[1])
except IndexError, e:
raise DecryptError("Error decrypting: inconsistent message")
def _serialize_key_value(self, key, key_info_element):
key_value = SubElement(key_info_element, ds_tag("KeyValue"))
if self.sign_alg.startswith("rsa-"):
rsa_key_value = SubElement(key_value, ds_tag("RSAKeyValue"))
modulus = SubElement(rsa_key_value, ds_tag("Modulus"))
modulus.text = ensure_str(b64encode(long_to_bytes(key.public_key().public_numbers().n)))
exponent = SubElement(rsa_key_value, ds_tag("Exponent"))
exponent.text = ensure_str(b64encode(long_to_bytes(key.public_key().public_numbers().e)))
elif self.sign_alg.startswith("dsa-"):
dsa_key_value = SubElement(key_value, ds_tag("DSAKeyValue"))
for field in "p", "q", "g", "y":
e = SubElement(dsa_key_value, ds_tag(field.upper()))
if field == "y":
key_params = key.public_key().public_numbers()
else:
key_params = key.parameters().parameter_numbers()
e.text = ensure_str(b64encode(long_to_bytes(getattr(key_params, field))))
elif self.sign_alg.startswith("ecdsa-"):
ec_key_value = SubElement(key_value, dsig11_tag("ECKeyValue"), nsmap=dict(dsig11=namespaces.dsig11))
named_curve = SubElement(ec_key_value, dsig11_tag("NamedCurve"),
URI=self.known_ecdsa_curve_oids[key.curve.name])
public_key = SubElement(ec_key_value, dsig11_tag("PublicKey"))
x = key.public_key().public_numbers().x
y = key.public_key().public_numbers().y
public_key.text = ensure_str(b64encode(long_to_bytes(4) + long_to_bytes(x) + long_to_bytes(y)))
def upload_file_to_s3():
# redirect_url = "http://songty.ucmpcs.org:8888/send_annotation_request"
redirect_url = str(request.url) + "/send_annotation_request"
print redirect_url
time = datetime.timedelta(days = 1) + datetime.datetime.today();
# define S3 policy document
policy = {"expiration":time.strftime('%Y-%m-%dT%H:%M:%S.000Z'),
"conditions":
[{"bucket":"gas-inputs"},
{"acl": "private"},
["starts-with", "$key", "songty/"],
["starts-with", "$success_action_redirect",redirect_url],
]
}
#https://docs.python.org/2/library/base64.html
Policy_Code = base64.b64encode(str(policy)).encode('utf8')
s3_key = str(uuid.uuid4())
session = botosession.get_session()
credentials = session.get_credentials()
access_key = credentials.access_key
secret_key = credentials.secret_key
#https://docs.python.org/2/library/hmac.html
my_hmac = hmac.new(secret_key.encode(),Policy_Code,hashlib.sha1)
digest = my_hmac.digest()
signature = base64.b64encode(digest)
tpl = template("upload.tpl",bucket_name = "gas-inputs",policy = Policy_Code,aws_key = access_key,signature = signature,redirect = redirect_url,s3_key = s3_key)
return tpl
def test_render(self):
xml = render_xml(self.template, {
'submission_id': 100,
'security_code': 'AB12CD3',
'rating_system': 'PEGI',
'release_date': datetime.date(2013, 11, 1),
'title': 'Twitter',
'company': 'Test User',
'rating': '16+',
'descriptors': u'N\xc3\xa3o h\xc3\xa1 inadequa\xc3\xa7\xc3\xb5es',
'interactive_elements': 'users interact'})
assert xml.startswith('<?xml version="1.0" encoding="utf-8"?>')
assert '<FIELD NAME="password" VALUE="s3kr3t"' in xml
assert '<FIELD NAME="storefront_company" VALUE="Test User"' in xml
assert '<FIELD NAME="platform" VALUE="Firefox"' in xml
assert '<FIELD NAME="submission_id" VALUE="100"' in xml
assert '<FIELD NAME="security_code" VALUE="AB12CD3"' in xml
assert '<FIELD NAME="rating_system" VALUE="PEGI"' in xml
assert '<FIELD NAME="release_date" VALUE="2013-11-01"' in xml
assert '<FIELD NAME="storefront_title" VALUE="Twitter"' in xml
assert '<FIELD NAME="storefront_rating" VALUE="16+"' in xml
assert ('<FIELD NAME="storefront_descriptors" '
u'VALUE="N\xc3\xa3o h\xc3\xa1 inadequa\xc3\xa7\xc3\xb5es"'
in xml)
assert ('<FIELD NAME="storefront_interactive_elements" '
'VALUE="users interact"') in xml
# The client base64 encodes these. Mimic what the client does here to
# ensure no unicode problems.
base64.b64encode(xml.encode('utf-8'))
def clean_keytab_file(self):
keytab_file = self.cleaned_data.get("keytab_file", None)
if not keytab_file:
raise forms.ValidationError(_("A keytab is required."))
encoded = None
if hasattr(keytab_file, 'temporary_file_path'):
filename = keytab_file.temporary_file_path()
with open(filename, "r") as f:
keytab_contents = f.read()
encoded = base64.b64encode(keytab_contents)
f.close()
else:
filename = tempfile.mktemp(dir='/tmp')
with open(filename, 'wb+') as f:
for c in keytab_file.chunks():
f.write(c)
f.close()
with open(filename, "r") as f:
keytab_contents = f.read()
encoded = base64.b64encode(keytab_contents)
f.close()
os.unlink(filename)
return encoded
def dump(module_data, d):
if module_data['mongo']:
module_data['db'].insert(d)
if module_data['carve_request'] and 'body' in d['request']:
chop.prnt("DUMPING REQUEST: %s (%i)" % (sanitize_filename(d['request']['uri']['path'][1:] + '.request.' + str(module_data['counter'])), len(d['request']['body'])))
chop.savefile(sanitize_filename(d['request']['uri']['path'][1:] + '.request.' + str(module_data['counter'])), d['request']['body'])
module_data['counter'] += 1
if module_data['carve_response'] and 'body' in d['response']:
chop.prnt("DUMPING RESPONSE: %s (%i)" % (sanitize_filename(d['request']['uri']['path'][1:] + '.response.' + str(module_data['counter'])), len(d['response']['body'])))
chop.savefile(sanitize_filename(d['request']['uri']['path'][1:] + '.response.' + str(module_data['counter'])), d['response']['body'])
module_data['counter'] += 1
# Convert the body to base64 encoded data, if it exists.
if 'body' in d['request']:
d['request']['body'] = b64encode(d['request']['body'])
d['request']['body_encoding'] = 'base64'
if 'body' in d['response']:
d['response']['body'] = b64encode(d['response']['body'])
d['response']['body_encoding'] = 'base64'
chop.prnt(d)
chop.json(d)
# In case pipelining is going on remove these.
d['request'] = { 'headers': {} }
d['response'] = { 'headers': {} }
def test_disks_flag(self):
# specifying a EBS mount or PD mount is only valid for EC2/Euca/GCE, so
# fail on a cluster deployment.
argv = self.cluster_argv[:] + ["--disks", "ABCDFEG"]
self.assertRaises(BadConfigurationException, ParseArgs, argv, self.function)
# if we get a --disk flag, fail if it's not a dict (after base64, yaml load)
bad_disks_layout = yaml.load("""
public1,
""")
base64ed_bad_disks = base64.b64encode(yaml.dump(bad_disks_layout))
cloud_argv1 = self.cloud_argv[:] + ["--disks", base64ed_bad_disks]
self.assertRaises(BadConfigurationException, ParseArgs, cloud_argv1,
self.function)
# passing in a dict should be fine, and result in us seeing the same value
# for --disks that we passed in.
disks = {'public1' : 'vol-ABCDEFG'}
good_disks_layout = yaml.load("""
public1 : vol-ABCDEFG
""")
base64ed_good_disks = base64.b64encode(yaml.dump(good_disks_layout))
cloud_argv2 = self.cloud_argv[:] + ["--disks", base64ed_good_disks]
actual = ParseArgs(cloud_argv2, self.function).args
self.assertEquals(disks, actual.disks)
def do_GET(self):
"""Sends commands b64 encoded in HTTP responses
"""
global last_command, output_ready,command_ready,password, salt
if ((self.client_address[0] == socket.gethostbyname(host)) and command_ready):
self.send_response(200) # begin sending response
if encrypt:
salt = self.headers["Content-Salt"].strip() # extract the salt the client is using
if verbose: print "received salt from client: "+salt
hasher = SHA.new() # new hasher
hasher.update(password + salt) # create the hash of the string passwordsalt
rc4 = ARC4.new(hasher.hexdigest()) # use the hash for password to avoid weak key scheduling
self.end_headers() # end of response headers
self.wfile.write(base64.b64encode(rc4.encrypt(last_command))) # send payload
else:
# send payload without encryption
self.end_headers()
self.wfile.write(base64.b64encode(last_command))
command_ready=False # wait for next command
else:
# GET does not come from the client we are currently listening to or there is no command available yet
self.send_response(200) # send empty response and end
self.send_header("Content-Type","0") # no command issued
self.end_headers()
# Check special header to know client current polling period
if "Next-Polling-In" in self.headers:
global next_polling,timestamp,client_sync
next_polling = self.headers["Next-Polling-In"] # so the server can calculate roughly next polling
# set the time of last request
timestamp = int(time.time())
client_sync = True
def setUp(self):
self.redirect_guid = JOB['guid']
self.redirect = RedirectFactory(buid=JOB['buid'],
guid='{%s}' %
uuid.UUID(self.redirect_guid))
self.password = '******'
self.user = User.objects.create(email='*****@*****.**')
self.user.set_password(self.password)
self.user.save()
self.contact = CompanyEmail.objects.create(
buid=self.redirect.buid,
email=self.user.email)
self.email = self.user.email.replace('@', '%40')
self.auth = {
'bad': [
'',
'Basic %s' % base64.b64encode('bad%40email:wrong_pass')],
'good':
'Basic %s' % base64.b64encode('%s:%s' % (self.user.email.\
replace('@', '%40'),
self.password))}
self.post_dict = {'to': '*****@*****.**',
'from': '*****@*****.**',
'text': 'This address does not contain a valid guid',
'html': '',
'subject': 'Bad Email',
'attachments': 0}
self.r = Replacer()
self.r.replace('pysolr.Solr.search', mock_search)
def save(self):
buffer = ""
# Save class watermarks
if self.__wms["SSS_CLASS"] != None:
sss = self.__wms["SSS_CLASS"]
buffer += '<class name="%s">\n' % self.__class_name
buffer += "<threshold>%d</threshold>\n" % (sss.get_threshold())
buffer += "<sss>%s</sss>\n" % (base64.b64encode(cPickle.dumps(sss.get_y())))
for j in self.__wms["CLASS"]:
buffer += '<wm type="%s">%s</wm>\n' % (j[0], base64.b64encode(cPickle.dumps(j[1].get_export_context())))
buffer += "</class>\n"
# Save methods watermarks
for i in self.__wms["SSS_METHODS"]:
sss = self.__wms["SSS_METHODS"][i]
buffer += '<method class="%s" name="%s" descriptor="%s">\n' % (
i.get_class_name(),
escape(i.get_name()),
i.get_descriptor(),
)
buffer += "<threshold>%d</threshold>\n" % (sss.get_threshold())
buffer += "<sss>%s</sss>\n" % (base64.b64encode(cPickle.dumps(sss.get_y())))
for j in self.__wms["METHODS"][i]:
buffer += '<wm type="%s">%s</wm>\n' % (j[0], base64.b64encode(cPickle.dumps(j[1].get_export_context())))
buffer += "</method>\n"
return buffer
def make_request(self, method, path, params={}, body="", username=None, password=None):
headers = {
'User-Agent': 'createsend-python-%s' % __version__,
'Content-Type': 'application/json; charset=utf-8',
'Accept-Encoding' : 'gzip, deflate' }
parsed_base_uri = urlparse(CreateSend.base_uri)
"""username and password should only be set when it is intended that
the default basic authentication mechanism using the API key be
overridden (e.g. when using the apikey route with username and password)."""
if username and password:
headers['Authorization'] = "Basic %s" % base64.b64encode("%s:%s" % (username, password))
else:
headers['Authorization'] = "Basic %s" % base64.b64encode("%s:x" % self.api_key)
"""If in fake web mode (i.e. self.stub_request has been called),
self.faker should be set, and this request should be treated as a fake."""
if self.fake_web:
# Check that the actual url which would be requested matches self.faker.url.
actual_url = "http://%s%s" % (parsed_base_uri.netloc, self.build_url(parsed_base_uri, path, params))
if self.faker.url != actual_url:
raise Exception("Faker's expected URL (%s) doesn't match actual URL (%s)" % (self.faker.url, actual_url))
data = self.faker.open() if self.faker else ''
status = self.faker.status if (self.faker and self.faker.status) else 200
return self.handle_response(status, data)
c = httplib.HTTPConnection(parsed_base_uri.netloc)
c.request(method, self.build_url(parsed_base_uri, path, params), body, headers)
response = c.getresponse()
if response.getheader('content-encoding', '') == 'gzip':
data = gzip.GzipFile(fileobj=StringIO(response.read())).read()
else:
data = response.read()
c.close()
return self.handle_response(response.status, data)
def _buildConsistencyProof(self, ledgerType, seqNoStart, seqNoEnd):
ledger = self.ledgers[ledgerType]["ledger"]
ledgerSize = ledger.size
if seqNoStart > ledgerSize:
logger.error("{} cannot build consistency proof from {} since its "
"ledger size is {}".format(self, seqNoStart,
ledgerSize))
return
if seqNoEnd > ledgerSize:
logger.error("{} cannot build consistency proof till {} since its "
"ledger size is {}".format(self, seqNoEnd, ledgerSize))
return
if seqNoStart == 0:
# Consistency proof for an empty tree cannot exist. Using the root
# hash now so that the node which is behind can verify that
# TODO: Make this an empty list
oldRoot = ledger.tree.root_hash
proof = [oldRoot, ]
else:
proof = ledger.tree.consistency_proof(seqNoStart, seqNoEnd)
oldRoot = ledger.tree.merkle_tree_hash(0, seqNoStart)
newRoot = ledger.tree.merkle_tree_hash(0, seqNoEnd)
return ConsistencyProof(
ledgerType,
seqNoStart,
seqNoEnd,
b64encode(oldRoot).decode(),
b64encode(newRoot).decode(),
[b64encode(p).decode() for p in
proof]
)
def toString(self):
s = ('<?xml version="1.0" encoding="UTF-8"?>' + "\n\n" + \
'<request seq="%s" type="%s">' + "\n") % (int(self.seqnum), self.reqtype)
if self.fields:
fieldstrings = {}
fieldcontents = {}
for field, attr in self.fields:
content = None
if not field in fieldstrings:
fieldstrings[field] = "\t<" + field
if attr and attr != "":
if attr.lower() == 'content':
fieldcontents[field] = self.fields[(field, attr)]
else:
fieldstrings[field] += ' %s="%s"' % (attr, self.fields[(field, attr)])
for field in fieldstrings.keys():
fieldstrings[field] += ">"
if field in fieldcontents:
fieldstrings[field] += "<![CDATA[%s]]>" % (base64.b64encode(fieldcontents[field]))
fieldstrings[field] += "</%s>\n" % (field)
s += fieldstrings[field]
if self.content:
s += "\t<content><![CDATA[%s]]></content>\n" % (base64.b64encode(self.content))
s += '</request>' + "\n"
s = "%d\n\n%s" % (len(s), s)
return s
def configure(self):
EucalyptusRequest.configure(self)
if self.args.get('user_data'):
if os.path.isfile(self.args['user_data']):
raise ArgumentError(
'argument -d/--user-data: to pass the contents of a file '
'as user data, use -f/--user-data-file. To pass the '
"literal value '{0}' as user data even though it matches "
'the name of a file, use --user-data-force.')
else:
self.params['UserData'] = base64.b64encode(
self.args['user_data'])
elif self.args.get('user_data_force'):
self.params['UserData'] = base64.b64encode(
self.args['user_data_force'])
elif self.args.get('user_data_file'):
with open(self.args['user_data_file']) as user_data_file:
self.params['UserData'] = base64.b64encode(
user_data_file.read())
if self.args.get('KeyName') is None:
default_key_name = self.config.get_region_option(
'ec2-default-keypair')
if default_key_name:
self.log.info("using default key pair '%s'", default_key_name)
self.params['KeyName'] = default_key_name
def _decrypt_image(self, context, encrypted_filename, encrypted_key,
encrypted_iv, decrypted_filename):
elevated = context.elevated()
try:
key = self.cert_rpcapi.decrypt_text(elevated,
project_id=context.project_id,
text=base64.b64encode(encrypted_key))
except Exception as exc:
msg = _('Failed to decrypt private key: %s') % exc
raise exception.NovaException(msg)
try:
iv = self.cert_rpcapi.decrypt_text(elevated,
project_id=context.project_id,
text=base64.b64encode(encrypted_iv))
except Exception as exc:
raise exception.NovaException(_('Failed to decrypt initialization '
'vector: %s') % exc)
try:
utils.execute('openssl', 'enc',
'-d', '-aes-128-cbc',
'-in', '%s' % (encrypted_filename,),
'-K', '%s' % (key,),
'-iv', '%s' % (iv,),
'-out', '%s' % (decrypted_filename,))
except processutils.ProcessExecutionError as exc:
raise exception.NovaException(_('Failed to decrypt image file '
'%(image_file)s: %(err)s') %
{'image_file': encrypted_filename,
'err': exc.stdout})
def genHawkSignature(method, urlStr, bodyHash, extra, secret,
now=None, nonce=None, ctype="application/json"):
""" Generate a HAWK signature from the content to be sent
"""
url = urlparse.urlparse(urlStr)
path = url.path
host = url.hostname
port = url.port
if port is None:
port = 80
if nonce is None:
nonce = os.urandom(5).encode("hex")
if now is None:
now = int(time.time())
marshalStr = "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n" % (
"hawk.1.header",
now,
nonce,
method.upper(),
path,
host.lower(),
port,
bodyHash,
extra)
print "Marshal Str: <<%s>>\nSecret: <<%s>>\n" % (marshalStr, secret)
mac = hmac.new(secret.encode("utf-8"),
marshalStr.encode("utf-8"),
digestmod=hashlib.sha256).digest()
print "mac: <<" + ','.join([str(ord(elem)) for elem in mac]) + ">>\n"
print "mac: " + base64.b64encode(mac) + "\n"
return now, nonce, base64.b64encode(mac)
def image(img, to_bgr=True, encoding='jpg', **kwargs):
""" image(img, [win, title, labels, width])
to_bgr: converts to bgr, if encoded as rgb (default True).
encoding: 'jpg' (default) or 'png'
"""
assert img.ndim == 2 or img.ndim == 3
win = kwargs.get('win') or uid()
if isinstance(img, list):
return images(img, kwargs)
# TODO: if img is a 3d tensor, then unstack it into a list of images
img = to_rgb(normalize(img, kwargs))
if to_bgr:
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
#pngbytes = png.encode(img.tostring(), img.shape[1], img.shape[0])
if encoding=='jpg':
jpgbytes = cv2.imencode('.jpg', img)[1]
imgdata = 'data:image/jpg;base64,' + base64.b64encode(jpgbytes).decode('ascii')
elif encoding=='png':
pngbytes = cv2.imencode('.png', img)[1]
imgdata = 'data:image/png;base64,' + base64.b64encode(pngbytes).decode('ascii')
else:
raise ValueError('unknown encoding')
send(command='image', id=win, src=imgdata,
labels=kwargs.get('labels'),
width=kwargs.get('width'),
title=kwargs.get('title'))
return win
def run(bucket, url, aws_secret):
# l = '--local' in sys.argv
# params = {
# 'url':(l and "http://localhost:8081" or "http://www.gtfs-data-exchange.com"),
# 'bucket':(l and 'gtfs-devel' or 'gtfs')
# }
policy_document = """
{"expiration": "2016-07-01T00:00:00Z",
"conditions": [
{"bucket": "%(bucket)s"},
["starts-with", "$key", "queue/"],
{"acl": "private"},
{"success_action_redirect": "%(url)s/queue"},
["eq", "$Content-Type", "application/zip"],
["content-length-range", 0, 31457280],
["starts-with","$x-amz-meta-user",""],
["starts-with","$x-amz-meta-comments",""]
]
}
""" % dict(
bucket=bucket,
url=url,
)
print policy_document
policy = base64.b64encode(policy_document.replace('\n','').strip())
signature = base64.b64encode(hmac.new(aws_secret.encode('utf-8'), policy, hashlib.sha1).digest())
print policy
print "-"*50
print signature
def test_existing_ovf_diff(self):
# waagent/SharedConfig must be removed if ovfenv is found elsewhere
# 'get_data' should remove SharedConfig.xml in /var/lib/waagent
# if ovf-env.xml differs.
cached_ovfenv = construct_valid_ovf_env(
{'userdata': base64.b64encode("FOO_USERDATA")})
new_ovfenv = construct_valid_ovf_env(
{'userdata': base64.b64encode("NEW_USERDATA")})
populate_dir(self.waagent_d,
{'ovf-env.xml': cached_ovfenv,
'SharedConfig.xml': "mysharedconfigxml",
'otherfile': 'otherfilecontent'})
dsrc = self._get_ds({'ovfcontent': new_ovfenv})
ret = dsrc.get_data()
self.assertTrue(ret)
self.assertEqual(dsrc.userdata_raw, "NEW_USERDATA")
self.assertTrue(os.path.exists(
os.path.join(self.waagent_d, 'otherfile')))
self.assertFalse(
os.path.exists(os.path.join(self.waagent_d, 'SharedConfig.xml')))
self.assertTrue(
os.path.exists(os.path.join(self.waagent_d, 'ovf-env.xml')))
self.assertEqual(new_ovfenv,
load_file(os.path.join(self.waagent_d, 'ovf-env.xml')))
def upload(self, lock):
self.__log.debug("1) calling upload() for " + self.img_path)
self.lock = lock
sha256 = hashlib.sha256()
fp = open(self.img_path, 'rb')
try:
im = Image.open(self.img_path)
im.thumbnail((128, 128), Image.ANTIALIAS)
output = BytesIO()
im.convert("RGB")
im.save(output, format='JPEG')
output.seek(0)
self.b64preview = base64.b64encode(output.read())
sha256.update(fp.read())
_hash = base64.b64encode(sha256.digest())
self.size = os.path.getsize(self.img_path)
self.methods_interface.call("media_requestUpload", (_hash, self.mtype, self.size))
fp.close()
except IOError:
self.__log.error("Caught an IOError. cancelling upload...")
self.cleanup()
except:
self.__log.error("Caught a generic error in file upload...")
self.cleanup()
def setup_module(module):
try:
shutil.rmtree('store')
except:
pass # !!!
config['server_host'] = {
'host': 'our_test_domain',
'port': '8001',
'scheme': 'http',
}
from tiddlyweb.web import serve
# we have to have a function that returns the callable,
# Selector just _is_ the callable
def app_fn():
return serve.load_app()
#wsgi_intercept.debuglevel = 1
httplib2_intercept.install()
wsgi_intercept.add_wsgi_intercept('our_test_domain', 8001, app_fn)
environ = {'tiddlyweb.config': config}
module.store = Store(config['server_store'][0], config['server_store'][1], environ)
admin = User('admin')
admin.add_role('ADMIN')
admin.set_password('spank')
module.store.put(admin)
module.admin_authorization = b64encode('admin:spank')
module.user_authorization = b64encode('cdent:pigdog')
def put(self):
allowed_routes = ['{{ function_name }}']
function = self.request.get('f')
input_source = self.request.get('input1')
json_data = {'f':function, 'input1':input_source}
output = ''
if self.request.get('output') == '':
key_length = 16 # for now, randomly generates keys 16 chars long
json_data['output'] = base64.b64encode(os.urandom(key_length)) # TODO - does this work in app engine?
else:
json_data['output'] = str(self.request.get('output'))
output = str(json_data['output'])
if function in allowed_routes:
url = '/' + function
logging.debug('starting a request for url ' + url)
queue.put_message(get_queue('tasks'), json.dumps(json_data))
key_length = 16
task_name = base64.b64encode(os.urandom(key_length))
result = {'result':'success', 'task_id':task_name, 'output':output, 'id':task_name}
logging.debug('result of job with input data' + str(json_data) + ' was ' + str(result))
self.response.out.write(json.dumps(result))
else:
reason = 'Cannot add a task for function type ' + str(function)
result ={'result':'failure', 'reason':reason}
self.response.out.write(json.dumps(result))
def register():
if request.method == 'POST':
if (not request.form['username'] or not request.form['password']
or not request.form['email'] or request.form['hp']
or request.form['password'] != request.form['password2']):
flash('All fields are required.')
else:
username = request.form['username']
password = request.form['password']
email = request.form['email']
# TODO validate/confirm email - #29 - Makyo
result = g.db.execute('select count(*) from auth_users where '
'username = ? or email = ?',
[username, email])
if result.fetchone()[0] != 0:
flash('That username or email is already in use. Perhaps you '
'want to login instead?')
return render_template('user_management/register.html')
# 64 bytes of salt since our hash is 64 bytes long; perhaps
# overkill but shouldn't take too long to generate
hashword, salt = generate_hashword(password)
g.db.execute(
'insert into auth_users (username, hashword, salt, email) '
'values (?, ?, ?, ?)',
[username,
base64.b64encode(hashword),
base64.b64encode(salt),
email])
g.db.commit()
_login(username)
flash('Thank you for registering! You are now logged in.')
return redirect('/')
return render_template('user_management/register.html')
def _test_etag_is_at_not_duplicated(self, method):
# verify only one occurrence of X-Object-Sysmeta-Crypto-Etag-Mac in
# X-Backend-Etag-Is-At
key = fetch_crypto_keys()['object']
env = {CRYPTO_KEY_CALLBACK: fetch_crypto_keys}
req = Request.blank(
'/v1/a/c/o', environ=env, method=method,
headers={'If-Match': '"an etag"',
'If-None-Match': '"another etag"'})
self.app.register(method, '/v1/a/c/o', HTTPOk, {})
resp = req.get_response(self.encrypter)
self.assertEqual('200 OK', resp.status)
self.assertEqual(1, len(self.app.calls), self.app.calls)
self.assertEqual(method, self.app.calls[0][0])
actual_headers = self.app.headers[0]
self.assertIn('X-Backend-Etag-Is-At', actual_headers)
self.assertEqual('X-Object-Sysmeta-Crypto-Etag-Mac',
actual_headers['X-Backend-Etag-Is-At'])
self.assertIn('"%s"' % base64.b64encode(
hmac.new(key, 'an etag', hashlib.sha256).digest()),
actual_headers['If-Match'])
self.assertIn('"another etag"', actual_headers['If-None-Match'])
self.assertIn('"%s"' % base64.b64encode(
hmac.new(key, 'another etag', hashlib.sha256).digest()),
actual_headers['If-None-Match'])
def redirect_to_custom_form(request, auth_entry, details, kwargs):
"""
If auth_entry is found in AUTH_ENTRY_CUSTOM, this is used to send provider
data to an external server's registration/login page.
The data is sent as a base64-encoded values in a POST request and includes
a cryptographic checksum in case the integrity of the data is important.
"""
backend_name = request.backend.name
provider_id = provider.Registry.get_from_pipeline({'backend': backend_name, 'kwargs': kwargs}).provider_id
form_info = AUTH_ENTRY_CUSTOM[auth_entry]
secret_key = form_info['secret_key']
if isinstance(secret_key, unicode):
secret_key = secret_key.encode('utf-8')
custom_form_url = form_info['url']
data_str = json.dumps({
"auth_entry": auth_entry,
"backend_name": backend_name,
"provider_id": provider_id,
"user_details": details,
})
digest = hmac.new(secret_key, msg=data_str, digestmod=hashlib.sha256).digest()
# Store the data in the session temporarily, then redirect to a page that will POST it to
# the custom login/register page.
request.session['tpa_custom_auth_entry_data'] = {
'data': base64.b64encode(data_str),
'hmac': base64.b64encode(digest),
'post_url': custom_form_url,
}
return redirect(reverse('tpa_post_to_custom_auth_form'))
def encode(self, data):
d = {
'salt': base64.b64encode(Rand.rand_bytes(8)),
'digest': 'sha1',
'validFrom': self._formatDate(datetime.datetime.utcnow()),
'validTo': self._formatDate(
datetime.datetime.utcnow() + datetime.timedelta(
seconds=self._lifetime
)
),
'data': data
}
self._pkey.reset_context(md=d['digest'])
self._pkey.sign_init()
fields = []
for k, v in d.items():
fields.append(k)
self._pkey.sign_update(v)
d['signedFields'] = ','.join(fields)
d['signature'] = base64.b64encode(self._pkey.sign_final())
d['certificate'] = self._x509.as_pem()
return base64.b64encode(json.dumps(d))
def getRss():
codert = urllib.urlopen("http://actualidad.rt.com/feeds/all.rss?rss=1")
codetn = urllib.urlopen("http://www.tn.com.ar/rss.xml")
codeinfobae = urllib.urlopen("http://cdn01.ib.infobae.com/adjuntos/162/rss/Infobae.xml")
#codetelam = urllib.urlopen("http://www.telam.com.ar/rss2/ultimasnoticas.xml")
jsonrt = {"rss" : base64.b64encode(codert.read())}
filert = open("../servicios/rt/serv.json", 'w')
filert.write(json.dumps(jsonrt))
filert.close()
jsontn = {"rss" : base64.b64encode(codetn.read())}
filetn = open("../servicios/tn/serv.json", 'w')
filetn.write(json.dumps(jsontn))
filetn.close()
jsoninfobae = {"rss" : base64.b64encode(codeinfobae.read())}
fileinfobae = open("../servicios/infobae/serv.json", 'w')
fileinfobae.write(json.dumps(jsoninfobae))
filert.close()
'''filetelam = open("../servicios/telam/rss.xml", 'w')
filetelam.write(codetelam.read())
filetelam.close()'''
print getTime(),"[RSS] RSS's actualizados"
threading.Timer(300.0, getRss).start()
def run_test(self):
#################################################
# lowlevel check for http persistent connection #
#################################################
url = urlparse.urlparse(self.nodes[0].url)
authpair = url.username + ':' + url.password
headers = {"Authorization": "Basic " + base64.b64encode(authpair)}
conn = httplib.HTTPConnection(url.hostname, url.port)
conn.connect()
conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
out1 = conn.getresponse().read()
assert_equal('"error":null' in out1, True)
assert_equal(conn.sock!=None, True) #according to http/1.1 connection must still be open!
#send 2nd request without closing connection
conn.request('POST', '/', '{"method": "getchaintips"}', headers)
out2 = conn.getresponse().read()
assert_equal('"error":null' in out1, True) #must also response with a correct json-rpc message
assert_equal(conn.sock!=None, True) #according to http/1.1 connection must still be open!
conn.close()
#same should be if we add keep-alive because this should be the std. behaviour
headers = {"Authorization": "Basic " + base64.b64encode(authpair), "Connection": "keep-alive"}
conn = httplib.HTTPConnection(url.hostname, url.port)
conn.connect()
conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
out1 = conn.getresponse().read()
assert_equal('"error":null' in out1, True)
assert_equal(conn.sock!=None, True) #according to http/1.1 connection must still be open!
#send 2nd request without closing connection
conn.request('POST', '/', '{"method": "getchaintips"}', headers)
out2 = conn.getresponse().read()
assert_equal('"error":null' in out1, True) #must also response with a correct json-rpc message
assert_equal(conn.sock!=None, True) #according to http/1.1 connection must still be open!
conn.close()
#now do the same with "Connection: close"
headers = {"Authorization": "Basic " + base64.b64encode(authpair), "Connection":"close"}
conn = httplib.HTTPConnection(url.hostname, url.port)
conn.connect()
conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
out1 = conn.getresponse().read()
assert_equal('"error":null' in out1, True)
assert_equal(conn.sock!=None, False) #now the connection must be closed after the response
#node1 (2nd node) is running with disabled keep-alive option
urlNode1 = urlparse.urlparse(self.nodes[1].url)
authpair = urlNode1.username + ':' + urlNode1.password
headers = {"Authorization": "Basic " + base64.b64encode(authpair)}
conn = httplib.HTTPConnection(urlNode1.hostname, urlNode1.port)
conn.connect()
conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
out1 = conn.getresponse().read()
assert_equal('"error":null' in out1, True)
#node2 (third node) is running with standard keep-alive parameters which means keep-alive is on
urlNode2 = urlparse.urlparse(self.nodes[2].url)
authpair = urlNode2.username + ':' + urlNode2.password
headers = {"Authorization": "Basic " + base64.b64encode(authpair)}
conn = httplib.HTTPConnection(urlNode2.hostname, urlNode2.port)
conn.connect()
conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
out1 = conn.getresponse().read()
assert_equal('"error":null' in out1, True)
assert_equal(conn.sock!=None, True) #connection must be closed because carboncoind should use keep-alive by default
# Check excessive request size
conn = httplib.HTTPConnection(urlNode2.hostname, urlNode2.port)
conn.connect()
conn.request('GET', '/' + ('x'*1000), '', headers)
out1 = conn.getresponse()
assert_equal(out1.status, httplib.NOT_FOUND)
conn = httplib.HTTPConnection(urlNode2.hostname, urlNode2.port)
conn.connect()
conn.request('GET', '/' + ('x'*10000), '', headers)
out1 = conn.getresponse()
assert_equal(out1.status, httplib.BAD_REQUEST)
def __init__(self, *args, **kwargs):
self.ref_func = kwargs.pop("ref_func", None)
self.ref_version = kwargs.pop("ref_version", None)
self.wrapper = kwargs.pop("wrapper", None)
self.id_func = kwargs.pop("id_func", lambda: b64encode(uuid4().bytes))
super(NodeField, self).__init__(*args, **kwargs)
def getByte(path):
with open(path, 'rb') as f:
img_byte = base64.b64encode(f.read())
img_str = img_byte.decode('ascii')
return img_str
def pk_from_raw(raw: dict) -> str:
unique = raw.get("presentation", str(raw["idReponse"]))
return base64.b64encode(unique.encode()).decode()
def plot_mode(data, countries, fit, mode):
"""
plot_mode:
Args: data [dict], countries [list], fit [bool], mode [str]
Returns: -
"""
days = len(data.get('Afghanistan'))
# initiate plot
fig = plt.figure()
ax1 = fig.add_subplot(1, 2, 1)
ax2 = fig.add_subplot(1, 2, 2)
ax1.set_yscale('log')
ax2.set_yscale('log')
x = np.arange(days)
x_fit = np.linspace(30, 80, 100)
xmini = 20
# extract y-axis data
for cnt in countries:
res = data.get(cnt)
if (res == None):
print(f'There is no country named {cnt}.'
f' Please refer to {COUNTRIES_CSV} for'
f' a list of valid countries.')
continue
mode_entries = []
for dt in res:
try:
mode_num = int(dt.get(mode))
except Exception as e:
print(f'Program breaks due to: {e}')
sys.exit(-1)
mode_entries.append(mode_num)
sing_entries = [mode_entries[i+1] - mode_entries[i] for i in range(len(mode_entries)-1)]
sing_entries.insert(0, mode_entries[0])
ax1.plot(x, mode_entries, 'o-', label=cnt)
ax2.scatter(x, sing_entries, label=cnt)
if fit:
param = np.polyfit(x, mode_entries, 3)
print(param)
y_fit = [param[0]*i**3 + param[1]*i**2 +param[2]*i + param[3] for i in x_fit]
ax1.plot(x_fit, y_fit, '--', label='fit '+cnt)
ax1.set_ylim(1.0, None)
ax2.set_ylim(1.0, None)
ax1.set_xlim(xmini, None)
ax2.set_xlim(xmini, None)
ax1.legend(frameon=False, loc='upper left', prop={'size': 6})
ax2.legend(frameon=False, loc='upper left', prop={'size': 6})
ax1.set_title(title_sum(mode))
ax2.set_title(title_sin(mode))
#plt.show()
plt.savefig('static/img/plot.png', bbox_inches='tight')
plt.savefig('static/img/plot.pdf', bbox_inches='tight')
# save byteimage
img = io.BytesIO()
plt.savefig(img, format='png')
img.seek(0)
plot_url = base64.b64encode(img.getvalue()).decode()
return(plot_url)
def encode_image(file_path):
with open(file_path, 'rb') as f:
image_content = f.read()
return base64.b64encode(image_content)
def test_decode(self):
obj = self.Cls()
data = b"\x80\x81\x82\x83"
obj.password = ProtectedPassword(base64.b64encode(data))
self.assertEqual(data, obj.password.value)
def bytes_b64_save(bytes_data, path, char_encoding='utf-8'):
"""Save bytes as a Base64 string to a file."""
b64_data = base64.b64encode(bytes_data)
b64_string = b64_data.decode(char_encoding)
string_save(b64_string, path)
def main(args):
"""
Store the defect results in the specified input list as bug reports in the
database.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
if not host_check.check_zlib():
raise Exception("zlib is not available on the system!")
# To ensure the help message prints the default folder properly,
# the 'default' for 'args.input' is a string, not a list.
# But we need lists for the foreach here to work.
if isinstance(args.input, str):
args.input = [args.input]
if 'name' not in args:
LOG.debug("Generating name for analysis...")
generated = __get_run_name(args.input)
if generated:
setattr(args, 'name', generated)
else:
LOG.error("No suitable name was found in the inputs for the "
"analysis run. Please specify one by passing argument "
"--name run_name in the invocation.")
sys.exit(2) # argparse returns error code 2 for bad invocations.
LOG.info("Storing analysis results for run '" + args.name + "'")
if 'force' in args:
LOG.info("argument --force was specified: the run with name '" +
args.name + "' will be deleted.")
# Setup connection to the remote server.
client = libclient.setup_client(args.product_url)
_, zip_file = tempfile.mkstemp('.zip')
LOG.debug("Will write mass store ZIP to '%s'...", zip_file)
try:
assemble_zip(args.input, zip_file, client)
if os.stat(zip_file).st_size > MAX_UPLOAD_SIZE:
LOG.error("The result list to upload is too big (max: %s).",
sizeof_fmt(MAX_UPLOAD_SIZE))
sys.exit(1)
with open(zip_file, 'rb') as zf:
b64zip = base64.b64encode(zf.read()).decode("utf-8")
context = webserver_context.get_context()
trim_path_prefixes = args.trim_path_prefix if \
'trim_path_prefix' in args else None
client.massStoreRun(args.name,
args.tag if 'tag' in args else None,
str(context.version),
b64zip,
'force' in args,
trim_path_prefixes)
# Storing analysis statistics if the server allows them.
if client.allowsStoringAnalysisStatistics():
storing_analysis_statistics(client, args.input, args.name)
LOG.info("Storage finished successfully.")
except RequestFailed as reqfail:
if reqfail.errorCode == ErrorCode.SOURCE_FILE:
header = ['File', 'Line', 'Checker name']
table = twodim_to_str('table',
header,
[c.split('|') for c in reqfail.extraInfo])
LOG.warning("Setting the review statuses for some reports failed "
"because of non valid source code comments: "
"%s\n %s", reqfail.message, table)
sys.exit(1)
except Exception as ex:
LOG.info("Storage failed: %s", str(ex))
sys.exit(1)
finally:
os.remove(zip_file)
def connect(self):
usrPass = "******" % (current_app.config['B1RESTUSR'], current_app.config['B1RESTPASS'])
b64Val = base64.b64encode(usrPass)
return {"Authorization": "Basic %s" % b64Val}
def support_process_ticket(self, **kwargs):
"""Adds the support ticket to the database and sends out emails to everyone following the support ticket category"""
values = {}
for field_name, field_value in kwargs.items():
values[field_name] = field_value
if values['my_gold'] != "256":
return "Bot Detected"
my_attachment = ""
file_name = ""
if "subcategory" in values:
sub_category = values['subcategory']
else:
sub_category = ""
# if "manager" in values:
# manager = values['manager']
# else:
# manager = ""
# pdb.set_trace()
if http.request.env.user.name != "Public user":
# import pdb
#pdb.set_trace()
portal_access_key = randint(1000000000,2000000000)
new_ticket_id = request.env['website.support.ticket'].sudo().create({'person_name':values['person_name'], 'category':values['category'],'priority_id':values['priority'], 'sub_category_id': sub_category, 'email':values['email'], 'description':values['description'], 'desk_number':values['desk_number'], 'subject':values['subject'], 'partner_id':http.request.env.user.partner_id.id, 'attachment': my_attachment, 'attachment_filename': file_name, 'portal_access_key': portal_access_key, 'channel': 'Website (User)'})
partner = http.request.env.user.partner_id
#Add to the communication history
partner.message_post(body="Customer " + partner.name + " has sent in a new support ticket", subject="New Support Ticket")
else:
search_partner = request.env['res.partner'].sudo().search([('email','=', values['email'] )])
if len(search_partner) > 0:
portal_access_key = randint(1000000000,2000000000)
new_ticket_id = request.env['website.support.ticket'].sudo().create({'person_name':values['person_name'], 'priority_id':values['priority'], 'category':values['category'], 'sub_category_id': sub_category, 'email':values['email'], 'description':values['description'], 'desk_number':values['desk_number'], 'subject':values['subject'], 'attachment': my_attachment, 'attachment_filename': file_name, 'partner_id':search_partner[0].id, 'portal_access_key': portal_access_key, 'channel': 'Website (Public)'})
else:
portal_access_key = randint(1000000000,2000000000)
new_ticket_id = request.env['website.support.ticket'].sudo().create({'person_name':values['person_name'], 'priority_id':values['priority'],'category':values['category'], 'sub_category_id': sub_category, 'email':values['email'], 'description':values['description'], 'desk_number':values['desk_number'], 'subject':values['subject'], 'attachment': my_attachment, 'attachment_filename': file_name, 'portal_access_key': portal_access_key, 'channel': 'Website (Public)'})
if "subcategory" in values:
#Also get the data from the extra fields
for extra_field in request.env['website.support.ticket.subcategory.field'].sudo().search([('wsts_id','=', int(sub_category) )]):
if "efield_" + str(extra_field.id) in values:
request.env['website.support.ticket.field'].sudo().create({'wst_id': new_ticket_id.id, 'name': extra_field.name, 'value': values["efield_" + str(extra_field.id)] })
else:
#All extra fields are required
return "Extra field is missing"
if 'file' in values:
for c_file in request.httprequest.files.getlist('file'):
data = c_file.read()
if c_file.filename:
request.env['ir.attachment'].sudo().create({
'name': c_file.filename,
'datas': base64.b64encode(data),
'datas_fname': c_file.filename,
'res_model': 'website.support.ticket',
'res_id': new_ticket_id.id
})
return werkzeug.utils.redirect("/support/ticket/thanks")
def get_table_download_link(df):
val = to_excel(df)
b64 = base64.b64encode(val)
return f'<a href="data:application/octet-stream;base64,{b64.decode()}" download="extract.xlsx">Download csv file</a>'
def encode_image(filename, datatype):
encoded = base64.b64encode(open(filename, "rb").read())
img = "data:{};base64,{}".format(datatype, encoded.decode())
return img
def _get_auth_string(user):
return 'Basic ' + base64.b64encode(b':'.join((
user.username.encode(),
user.username.encode()))
).strip().decode()
def encode_cram_md5(challenge, user, password):
challenge = base64.decodebytes(challenge)
response = user + b' ' + bytes(
hmac.HMAC(password, challenge).hexdigest(), 'utf-8')
return str(base64.b64encode(response), 'utf-8')
def _protect_str_vals(str_line):
r"""Parsing is greatly simplified if it can be assumed that key-value pairs
in the config match the regular expression '\w+\s*=\s*[\w.+-]+'. That is,
the values do not contain spaces, quotes, or escaped characters. This
function replaces values in the given `str_line` by a base-64 string
which is guaranteed to contain only letters and numbers.
Args:
str_line (str): Unprotected string to be parsed.
Returns:
tuple: Tuple containing
(str): Protected (base-64) string.
It is guaranteed to not be shorter than the original string.
(list): List of replacement tuples.
Examples:
>>> s_in = r'a = "text", '+"b = 'text'"
>>> print(s_in)
a = "text", b = 'text'
>>> s, r = _protect_str_vals(s_in)
>>> print(s)
a = InRleHQi, b = J3RleHQn
>>> print(r)
[('InRleHQi', '"text"'), ('J3RleHQn', "'text'")]
>>> s_in = r'a = this\ is\ an\ unquoted\ string, b = "text"'
>>> print(s_in)
a = this\ is\ an\ unquoted\ string, b = "text"
>>> s, r = _protect_str_vals(s_in)
>>> print(s)
a = dGhpc1wgaXNcIGFuXCB1bnF1b3RlZFwgc3RyaW5n, b = InRleHQi
>>> print(r)
[('InRleHQi', '"text"'), ('dGhpc1wgaXNcIGFuXCB1bnF1b3RlZFwgc3RyaW5n', 'this\\ is\\ an\\ unquoted\\ string')]
"""
# handle quoted strings
rx_dq = re.compile(r'"[^"\\]*(?:\\.[^"\\]*)*"') # search for ...
rx_sq = re.compile(r"'[^'\\]*(?:\\.[^'\\]*)*'") # ... balanced quotes
replacements = []
n_replacements = -1
while n_replacements < len(replacements):
n_replacements = len(replacements)
matches = rx_dq.findall(str_line)
matches.extend(rx_sq.findall(str_line))
for quoted_str in matches:
b64 = (
base64.b64encode(bytes(quoted_str, 'utf8'))
.decode('ascii')
.replace("=", '')
)
replacements.append((b64, quoted_str))
str_line = str_line.replace(quoted_str, b64)
# handle un-quoted, but escaped strings
rx_escaped_word = re.compile(
# any string of characters that does not include spaces or any
# of the characters ,:!&=\, except when they are escaped (preceded
# by a backslash)
r'(([^\s,:!&=\\]|\\\s|\\\\|\\,|\\:|\\!|\\&|\\=|\\n|\\r|\\t|\\b)+)'
)
rx_good_word = re.compile(r'^[\w.+-]+$')
for match in rx_escaped_word.finditer(str_line):
word = match.group(0)
if not (rx_good_word.match(word) or (word == '*')):
b64 = (
base64.b64encode(bytes(word, 'utf8'))
.decode('ascii')
.replace("=", '')
)
replacements.append((b64, word))
str_line = str_line.replace(word, b64)
return str_line, replacements
def generateVerificationCodeChallenge(verification_code: str) -> str:
encoded = sha512(verification_code.encode()).digest()
return b64encode(encoded, altchars=b"_-").decode()
def main():
warnings.warn("This script is aimed to demonstrate how to convert the\n"
"JSON file to a single image dataset, and not to handle\n"
"multiple JSON files to generate a real-use dataset.")
parser = argparse.ArgumentParser()
parser.add_argument('json_file')
parser.add_argument('-o', '--out', default=None)
args = parser.parse_args()
json_file = args.json_file
alist = os.listdir(json_file)
for i in range(0, len(alist)):
path = os.path.join(json_file, alist[i])
data = json.load(open(path))
out_dir = osp.basename(path).replace('.', '_')
out_dir = osp.join(osp.dirname(path), out_dir)
if not osp.exists(out_dir):
os.mkdir(out_dir)
if data['imageData']:
imageData = data['imageData']
else:
imagePath = os.path.join(os.path.dirname(path), data['imagePath'])
with open(imagePath, 'rb') as f:
imageData = f.read()
imageData = base64.b64encode(imageData).decode('utf-8')
img = utils.img_b64_to_arr(imageData)
label_name_to_value = {'_background_': 0}
for shape in data['shapes']:
label_name = shape['label']
if label_name in label_name_to_value:
label_value = label_name_to_value[label_name]
else:
label_value = len(label_name_to_value)
label_name_to_value[label_name] = label_value
# label_values must be dense
label_values, label_names = [], []
for ln, lv in sorted(label_name_to_value.items(), key=lambda x: x[1]):
label_values.append(lv)
label_names.append(ln)
assert label_values == list(range(len(label_values)))
lbl = utils.shapes_to_label(img.shape, data['shapes'], label_name_to_value)
captions = ['{}: {}'.format(lv, ln)
for ln, lv in label_name_to_value.items()]
lbl_viz = utils.draw_label(lbl, img, captions)
PIL.Image.fromarray(img).save(osp.join(out_dir, 'img.png'))
utils.lblsave(osp.join(out_dir, 'label.png'), lbl)
PIL.Image.fromarray(lbl_viz).save(osp.join(out_dir, 'label_viz.png'))
with open(osp.join(out_dir, 'label_names.txt'), 'w') as f:
for lbl_name in label_names:
f.write(lbl_name + '\n')
warnings.warn('info.yaml is being replaced by label_names.txt')
info = dict(label_names=label_names)
with open(osp.join(out_dir, 'info.yaml'), 'w') as f:
yaml.safe_dump(info, f, default_flow_style=False)
print('Saved to: %s' % out_dir)
def upload_files(self, images, location='Journal', stamp_no=None):
api = self._client._get_client(self._service)
loc = api.factory.create('AttachmentLocation')
if not hasattr(loc, location):
raise AttributeError('Location not supported')
if not stamp_no:
stamp_no = api.service.GetStampNo()
files = []
for frame_no, path in enumerate(images):
if path.lower().endswith('.jpeg') or path.lower().endswith('.jpg'):
filetype = 'Jpeg'
elif path.lower().endswith('.png'):
filetype = 'Png'
elif path.lower().endswith('.tif') or path.lower().endswith(
'.tiff'):
filetype = 'Tiff'
else:
raise AttributeError('Filetype not supported')
types = api.factory.create('ImageType')
if not hasattr(types, filetype):
raise AttributeError('Filetype not supported')
# load file from disk
with open(path, 'rb') as f:
content = f.read()
file_obj = api.service.Create(filetype)
file_obj.FrameInfo = api.factory.create('ArrayOfImageFrameInfo')
# create one frame per image
frame = api.factory.create('ImageFrameInfo')
frame.Id = 1
frame.Status = 0
frame.StampNo = stamp_no
# add the frame/image to the file object via the array of image frame info
file_obj.FrameInfo.ImageFrameInfo.append(frame)
# max chunk size
# max_length = api.service.GetMaxRequestLength()
max_length = 2000 * 1024
# upload the files
offset = 0
while offset <= len(content):
# extract next part of the content
part = content[offset:offset + max_length]
# part = part.encode('base64')
part = base64.b64encode(part)
if isinstance(part, bytes):
part = part.decode()
# print('uploading offset = {}, bytes = {}'.format(offset, len(part)))
api.service.AppendChunk(file_obj, part, offset)
offset += max_length
# print('All chunks uploaded OK')
api.service.Save(file_obj, location)
files.append(file_obj)
return dict(
# Id=file_obj.Id,
# Type=file_obj.Type,
StampNo=stamp_no,
Location=location,
)
def stringToBase64(s):
return base64.b64encode(s.encode('utf-8')).decode("utf-8")
def header(keyID,
privateKey,
reqURL,
reqMethod,
body=None,
algorithm="rsa-sha256"):
sign_list = []
url_parse = parse.urlparse(reqURL)
_header_field = ["(request-target)", "date", "host"]
_header = {
'host':
str(url_parse.netloc),
'user-agent':
'oracle-api/1.0',
'date':
str(datetime.datetime.utcnow().strftime(
"%a, %d %h %Y %H:%M:%S GMT")),
'accept':
'*/*',
'accept-encoding':
'',
}
sign_list.append(
str("(request-target): {} {}").format(
str(reqMethod).lower(),
parse.quote_plus(str("{}{}").format(
url_parse.path,
str("?{}").format(url_parse.query)
if url_parse.query else ""),
safe="/?=&~")))
if body is not None:
if isinstance(body, (dict, list)):
_body = json.dumps(body)
else:
_body = body
_header_field += [
"content-length", "content-type", "x-content-sha256"
]
_header['content-type'] = 'application/json'
_header['content-length'] = str(len(_body))
_header['x-content-sha256'] = str(
base64.b64encode(
hashlib.sha256(
_body.encode("utf-8")).digest()).decode("utf-8"))
sign_list += [
str("{}: {}").format(item, _header[item]) for item in _header_field
if "target" not in item
]
_signature = base64.b64encode(
rsa.sign(
str(str("\n").join(sign_list)).encode("utf-8"),
rsa.PrivateKey.load_pkcs1(
privateKey if isinstance(privateKey, bytes) else str(
privateKey).strip().encode("utf-8")),
str(algorithm).split("-", 1)[-1].upper().replace(
"SHA", "SHA-"))).decode("utf-8")
_header['authorization'] = str(
'Signature keyId="{}",algorithm="{}",signature="{}",headers="{}"'
).format(keyID, algorithm, _signature,
str(" ").join(_header_field))
return _header
timeout=None):
if not (name or id):
raise ZatoException(msg='Either name or id must be provided')
if name and output_repeated == ZATO_NOT_GIVEN:
output_repeated = name.lower().endswith('list')
if to_json:
payload = dumps(payload, default=default_json_handler)
id_, value = ('name', name) if name else ('id', id)
request = {
id_: value,
'payload': b64encode(payload.encode('utf8') if PY3 else payload),
'channel': channel,
'data_format': data_format,
'transport': transport,
'async': async,
'expiration': expiration,
'pid': pid,
'all_pids': all_pids,
'timeout': timeout,
}
return super(AnyServiceInvoker,
self).invoke(dumps(request, default=default_json_handler),
ServiceInvokeResponse, async, headers,
output_repeated)
def encode(data):
#[::-1] + id_generator()
return (base64.b64encode(str(data)))[::-1] + id_generator()
def image_encode_base64(self):
with open(self.path_to_save_image, 'rb') as f:
image = base64.b64encode(f.read())
base64_image = str(image)
return base64_image
os.environ["DATABASE_URI"] = os.environ["DATABASE_URL"]
for var in dir(config):
if var.startswith("__"): continue # Ignore dunders
if var in os.environ: setattr(config, var, os.environ[var])
else: failed.append(var)
if failed:
print("Required config variables %s not found - see config_sample.py" %
", ".join(failed),
file=sys.stderr)
sys.exit(1)
sys.modules["config"] = config # Make the config vars available elsewhere
import database
import utils
app = Flask(__name__)
app.secret_key = config.SESSION_SECRET or base64.b64encode(os.urandom(12))
scheduler = utils.Scheduler()
sockets = Sockets(app)
# Override Flask's forcing of Location headers to be absolute, since it
# gets stuff flat-out wrong. Also, the spec now says that relative
# headers are fine (and even when the spec said that the Location should
# to be absolute, everyone accepted relative URIs).
if os.environ.get("OVERRIDE_REDIRECT_HTTPS"):
from werkzeug.middleware.proxy_fix import ProxyFix
app.wsgi_app = ProxyFix(app.wsgi_app) # Grab info from Forwarded headers
_redirect = redirect
def redirect(*a, **kw):
resp = _redirect(*a, **kw)
resp.autocorrect_location_header = False
def _password_grant_auth_header(self):
user_pass = self.client.client_id + ':' + self.client.client_secret
auth = b'Basic ' + b64encode(user_pass.encode('utf-8'))
auth_header = {'HTTP_AUTHORIZATION': auth.decode('utf-8')}
return auth_header
def aes_encode(inputdes, key):
from Crypto.Cipher import AES
iv = str(bytearray(16))
cipher = AES.new(key[:16], AES.MODE_ECB, iv)
inputdes = force_bytes(inputdes)
return base64.b64encode(cipher.encrypt(pad(inputdes, 16)))
def __init__(self, host, port, username, password): authpair = "%s:%s" % (username, password) self.authhdr = "Basic %s" % (base64.b64encode(authpair)) self.conn = httplib.HTTPConnection(host, port, False, 30)
def rsa_encrypt(s, pubkey_str):
key = _str2key(pubkey_str)
modulus = int(key[0], 16)
exponent = int(key[1], 16)
pubkey = rsa.PublicKey(modulus, exponent)
return base64.b64encode(rsa.encrypt(s.encode(), pubkey)).decode()
def basic_auth(self):
"""Find basic auth, and returns base64 encoded
"""
credentials = "%s:%s" % (self.client_id, self.client_secret)
return base64.b64encode(credentials.encode('utf-8')).decode('utf-8').replace("\n", "")
