def get_image_for_ecu(ecu_serial): """ Intended to be called via XMLRPC by the Secondary client, either partial or full verification. Returns the following to the requesting Secondary: - filename of the firmware image assigned it by the Director and validated by the Primary's full verification (against both repositories, etc). The filename provided is relative to the targets directory. - binary image data for that file in xmlrpc.Binary format """ # Ensure serial is correct format & registered primary_ecu._check_ecu_serial(ecu_serial) image_fname = primary_ecu.get_image_fname_for_ecu(ecu_serial) if image_fname is None: print('ECU Serial ' + repr(ecu_serial) + ' requested an image, but this ' 'Primary has no update for that ECU.') return None, None assert os.path.exists(image_fname), 'File ' + repr(image_fname) + \ ' does not exist....' binary_data = xmlrpc_client.Binary(open(image_fname, 'rb').read()) print('Distributing image to ECU ' + repr(ecu_serial)) # Get relative filename (relative to the client targets directory) so that # it can be used as a TUF-style filepath within the targets namespace by # the Secondary. relative_fname = os.path.relpath( image_fname, os.path.join(primary_ecu.full_client_dir, 'targets')) return (relative_fname, binary_data)
def submit_vehicle_manifest_to_director(signed_vehicle_manifest=None): global most_recent_signed_vehicle_manifest if signed_vehicle_manifest is None: signed_vehicle_manifest = most_recent_signed_vehicle_manifest if tuf.conf.METADATA_FORMAT == 'der': # If we're working with DER ECU Manifests, check that the manifest to send # is a byte array, and encapsulate it in a Binary() object for XMLRPC # transmission. uptane.formats.DER_DATA_SCHEMA.check_match(signed_vehicle_manifest) signed_vehicle_manifest = xmlrpc_client.Binary(signed_vehicle_manifest) else: uptane.formats.SIGNABLE_VEHICLE_VERSION_MANIFEST_SCHEMA.check_match( signed_vehicle_manifest) server = xmlrpc_client.ServerProxy('http://' + str(demo.DIRECTOR_SERVER_HOST) + ':' + str(demo.DIRECTOR_SERVER_PORT)) print("Submitting the Primary's manifest to the Director.") server.submit_vehicle_manifest(primary_ecu.vin, primary_ecu.ecu_serial, signed_vehicle_manifest) print(GREEN + 'Submission of Vehicle Manifest complete.' + ENDCOLORS)
def load_torrent(proxy, ID, path): """Send a torrent to rtorrent and kick off the hash recheck""" logger = logging.getLogger(__name__) torrent = ptpapi.Torrent(ID=ID) torrent_data = torrent.download() data = bencode.bdecode(torrent_data) thash = metafile.info_hash(data) try: logger.debug(u"Testing for hash {0}".format( proxy.d.hash(thash, fail_silently=True))) logger.error( u"Hash {0} already exists in rtorrent as {1}, cannot load.".format( thash, proxy.d.name(thash))) return except (xmlrpc_client.Fault, xmlrpc.HashNotFound): pass proxy.load.raw('', xmlrpc_client.Binary(torrent_data)) # Wait until the torrent is loaded and available while True: sleep(1) try: proxy.d.hash(thash, fail_silently=True) break except (xmlrpc_client.Fault, xmlrpc.HashNotFound): pass logger.info(u"Torrent loaded at {0}".format(path)) proxy.d.custom.set(thash, 'tm_completed', str(int(time()))) proxy.d.directory.set(thash, path) proxy.d.check_hash(thash)
def attach_file(self, page, space, files, comments=None): existing_page = self.get_page(page, space) if comments is None: comments = {} for file_name, data in files.items(): try: self._api.removeAttachment(self.token, existing_page["id"], file_name) except xmlrpclib.Fault: self.debug("No existing attachment to replace") mime = guess_type(file_name)[0] if mime is None: self.warning("Could not determine MIME type of %s", file_name) mime = "application/binary" attachment = { "fileName": file_name, "contentType": mime, "comment": comments.get(file_name)} if isinstance(data, str) and os.path.exists(data): with open(data, "rb") as fin: data = fin.read() if not isinstance(data, bytes): raise TypeError("%s has the wrong data type \"%s\"" % (file_name, type(data))) try: self.debug("Calling addAttachment(%s, %s, %s, ...)", self.token, existing_page["id"], attachment) self._api.addAttachment( self.token, existing_page["id"], attachment, xmlrpclib.Binary(data)) self.debug("Uploaded %s", file_name) except xmlrpclib.Error: self.exception("Unable to attach %s", file_name)
def submit_ecu_manifest_to_primary(signed_ecu_manifest=None): global most_recent_signed_ecu_manifest if signed_ecu_manifest is None: signed_ecu_manifest = most_recent_signed_ecu_manifest if tuf.conf.METADATA_FORMAT == 'der': # TODO: Consider validation of DER manifests as well here. (Harder) # If we're using ASN.1/DER data, then we have to transmit this slightly # differently via XMLRPC, wrapped in a Binary object. signed_ecu_manifest = xmlrpc_client.Binary(signed_ecu_manifest) else: # Otherwise, we're working with standard Python dictionary data as # specified in uptane.formats. Validate and keep as-is. uptane.formats.SIGNABLE_ECU_VERSION_MANIFEST_SCHEMA.check_match( signed_ecu_manifest) server = xmlrpc_client.ServerProxy('http://' + str(_primary_host) + ':' + str(_primary_port)) #if not server.system.listMethods(): # raise Exception('Unable to connect to server.') server.submit_ecu_manifest(secondary_ecu.vin, secondary_ecu.ecu_serial, secondary_ecu.nonce_next, signed_ecu_manifest) # We don't switch to a new nonce for next time yet. That only happens when a # time attestation using that nonce is validated. # "Nonces" may be sent multiple times, but only validated once. secondary_ecu.set_nonce_as_sent()
def get_time_attestation_for_ecu(ecu_serial): """ """ I_TO_PRINT = TO_PRINT + uptane.YELLOW + '[get_time_attestation_for_ecu()]: ' + uptane.ENDCOLORS #TODO: Print to be deleted print( str('%s %s %s' % (I_TO_PRINT, 'Getting time attestation for ecu with ecu_serial:', ecu_serial))) #TODO: Until here # Ensure serial is correct format & registered primary_ecu._check_ecu_serial(ecu_serial) attestation = primary_ecu.get_last_timeserver_attestation() # If we're using ASN.1/DER, then the attestation is binary data we're about # to transmit via XMLRPC, so we should wrap it appropriately: if tuf.conf.METADATA_FORMAT == 'der': attestation = xmlrpc_client.Binary(attestation) #TODO: Print to be deleted print(str('%s %s' % (I_TO_PRINT, 'Returning attestation'))) #TODO: Until here return attestation
def _handle_binary(self, arg): try: if PY3: if not isinstance(arg, bytes): arg = bytes(arg, 'ascii') else: arg = str(arg) except UnicodeError: raise ValueError('Cannot represent %r as binary.' % arg) return xmlrpclib.Binary(arg)
def binarize_args(arg): # Converts recursively basestring arg into xmlrpclib.Binary, as they can # contain non-XML allowed characters if isinstance(arg, basestring): if isinstance(arg, unicode): arg = arg.encode('utf-8') return xmlrpclib.Binary(arg) if isinstance(arg, (list, tuple, set)): return list(map(binarize_args, arg)) if isinstance(arg, dict): return {k: binarize_args(v) for k, v in six.iteritems(arg)} return arg
def get_signed_time_der_wrapper(nonces): """ Encapsulates the binary data of the DER encoding of the timeserver attestation in an XMLPRC Binary object, for delivery via XMLRPC within the demo. This is only necessary when using DER format instead of the standard Uptane Python dictionary format. """ der_attestation = timeserver.get_signed_time_der(nonces) return xmlrpc_client.Binary(der_attestation)
def test_demo_timeserver(): """ Test the demo timeserver. # TODO: Consider moving these tests into a demo integration test module. """ I_TO_PRINT = TO_PRINT + uptane.YELLOW + '[test_demo_timeserver()]: ' + uptane.ENDCOLORS #TODO: Print to be deleted print(str('%s %s' % (I_TO_PRINT, 'Testing the Demo Timeserver'))) #TODO: Until here # Prepare to validate signatures. timeserver_key_pub = demo.import_public_key('timeserver') tuf.formats.ANYKEY_SCHEMA.check_match(timeserver_key_pub) # Fetch a normal signed time attestation, without ASN.1 format or DER # encoding, and validate the signature. signed_time = timeserver.get_signed_time([1, 2]) assert len( signed_time['signatures']) == 1, 'Unexpected number of signatures.' assert uptane.common.verify_signature_over_metadata( timeserver_key_pub, signed_time['signatures'][0], signed_time['signed'], DATATYPE_TIME_ATTESTATION, metadata_format='json' ), 'Demo Timeserver self-test fail: unable to verify signature over JSON.' # Fetch a DER-encoded converted-to-ASN.1 signed time attestation, with a # signature over the DER encoding. der_signed_time = timeserver.get_signed_time_der([2, 9, 151]) # Encapsulate that in a Binary object for XML-RPC. xb_der_signed_time = xmlrpc_client.Binary(der_signed_time) assert der_signed_time == xb_der_signed_time.data, \ 'Demo Timeserver self-test fail: xmlrpc Binary encapsulation issue' # Validate that signature. for pydict_again in [ asn1_codec.convert_signed_der_to_dersigned_json( der_signed_time, DATATYPE_TIME_ATTESTATION), asn1_codec.convert_signed_der_to_dersigned_json( xb_der_signed_time.data, DATATYPE_TIME_ATTESTATION) ]: assert uptane.common.verify_signature_over_metadata( timeserver_key_pub, pydict_again['signatures'][0], pydict_again['signed'], DATATYPE_TIME_ATTESTATION, metadata_format='der' ), 'Demo Timeserver self-test fail: unable to verify signature over DER'
def get_time_attestation_for_ecu(ecu_serial): """ """ # Ensure serial is correct format & registered primary_ecu._check_ecu_serial(ecu_serial) attestation = primary_ecu.get_last_timeserver_attestation() # If we're using ASN.1/DER, then the attestation is binary data we're about # to transmit via XMLRPC, so we should wrap it appropriately: if tuf.conf.METADATA_FORMAT == 'der': attestation = xmlrpc_client.Binary(attestation) return attestation
def run(self, *args, **kwargs): tasks = args self.set_hub(**kwargs) failed = False for task in tasks: task_name = os.path.basename(task) task_binary = xmlrpc_client.Binary(open(task, "rb").read()) print(task_name) try: print(self.hub.tasks.upload(task_name, task_binary)) except (KeyboardInterrupt, SystemExit): raise except Exception as ex: failed = True sys.stderr.write('Exception: %s\n' % ex) if failed: sys.exit(1)
def _AddFiles(self, files): """добавляет или изменяет файлы, необходимые для работы пакета принимает один параметр files - полностью идентичный одноименному параметру для JobPacket.AddJob""" if not isinstance(files, dict): files = dict((os.path.split(file)[-1], file) for file in files) for fname, fpath in files.items(): if not os.path.isfile(fpath): raise AttributeError("can't find file \"%s\"" % fpath) checksum = self._GetFileChecksum(fpath, self.conn.checksumDbPath) if not self._TryCheckBinaryAndLock(checksum, fpath): data = open(fpath, 'rb').read() checksum2 = hashlib.md5(data).hexdigest() if (checksum2 == checksum) or not self._TryCheckBinaryAndLock( checksum2, fpath): self.proxy.save_binary(xmlrpc_client.Binary(data)) checksum = checksum2 self.proxy.pck_add_binary(self.pck_id, fname, checksum)
def submit_vehicle_manifest_to_director(signed_vehicle_manifest=None): global most_recent_signed_vehicle_manifest I_TO_PRINT = TO_PRINT + uptane.YELLOW + '[submit_vehicle_manifest_to_director()]: ' + uptane.ENDCOLORS if signed_vehicle_manifest is None: signed_vehicle_manifest = most_recent_signed_vehicle_manifest if tuf.conf.METADATA_FORMAT == 'der': # If we're working with DER ECU Manifests, check that the manifest to send # is a byte array, and encapsulate it in a Binary() object for XMLRPC # transmission. uptane.formats.DER_DATA_SCHEMA.check_match(signed_vehicle_manifest) signed_vehicle_manifest = xmlrpc_client.Binary(signed_vehicle_manifest) else: uptane.formats.SIGNABLE_VEHICLE_VERSION_MANIFEST_SCHEMA.check_match( signed_vehicle_manifest) server = xmlrpc_client.ServerProxy('http://' + str(demo.DIRECTOR_SERVER_HOST) + ':' + str(demo.DIRECTOR_SERVER_PORT)) #TODO: Print to be deleted print( str('%s %s %s %s %s %s %s' % ( I_TO_PRINT, 'Connection already done. Submitting vehicle manifest to director with primary_ecu.vin:', primary_ecu.vin, 'primary_ecu.ecu_serial:', primary_ecu.ecu_serial, 'signed_vehicle_manifest:', '?'))) #TODO: Until here print("Submitting the Primary's manifest to the Director.") server.submit_vehicle_manifest(primary_ecu.vin, primary_ecu.ecu_serial, signed_vehicle_manifest) print(GREEN + 'Submission of Vehicle Manifest complete.' + ENDCOLORS) #TODO: Print to be deleted print(str('%s %s' % (I_TO_PRINT, 'Returning...')))
def get_signed_time_der_wrapper(nonces): """ Encapsulates the binary data of the DER encoding of the timeserver attestation in an XMLPRC Binary object, for delivery via XMLRPC within the demo. This is only necessary when using DER format instead of the standard Uptane Python dictionary format. """ I_TO_PRINT = TO_PRINT + uptane.YELLOW + '[get_signed_time_der_wrapper(nonces)]: ' + uptane.ENDCOLORS #TODO: Print to be deleted print( str('%s %s' % ( I_TO_PRINT, 'Encapsulates the binary data of the DER encoding of the timeserver attestation in an XMLPRC Binary Object, for delivery via XMLRPC within the demo.' ))) #TODO: Until here der_attestation = timeserver.get_signed_time_der(nonces) return xmlrpc_client.Binary(der_attestation)
def get_metadata_for_ecu(ecu_serial, force_partial_verification=False): """ Provides the current metadata a Secondary will need to validate updates. This takes two forms: - For Full Verification Secondaries (the norm): Send a zip archive of the most recent consistent set of the Primary's client metadata directory, containing the current, consistent metadata from all repositories used. - For Partial Verification Secondaries: Send the Director's Targets role file. <Arguments> ecu_serial the serial of the (Secondary) ECU for which to retrieve metadata force_partial_verification (optional: default False (Full)) If True, provides the partial metadata (the Director's Targets role file), else provides the full metadata archive. Which metadata is provided (full vs partial) is entirely determined by force_partial_verification, which should be renamed to partial_verification, but is not yet because there are other branches to be merged that call these. # TODO: rename force_partial_verification. <Exceptions> uptane.Error if there is no metadata to distribute """ I_TO_PRINT = TO_PRINT + uptane.YELLOW + '[get_metadata_for_ecu()]: ' + uptane.ENDCOLORS #TODO: Print to be deleted print( str('%s %s %s' % (I_TO_PRINT, 'Getting metadata for ecu with ecu_serial:', ecu_serial))) #TODO: Until here # Ensure serial is correct format & registered primary_ecu._check_ecu_serial(ecu_serial) # The filename of the file to return. fname = None if force_partial_verification: fname = primary_ecu.get_partial_metadata_fname() else: # Note that in Python 2.7.4 and later, unzipping should prevent files from # being created outside of the target extraction directory. There are other # security concerns (such as zip bombs). The security of archive use in # your environment should be carefully considered. fname = primary_ecu.get_full_metadata_archive_fname() if not os.path.exists(fname): raise uptane.Error( 'Primary has no metadata to distribute to Secondary "' + ecu_serial + '". Missing filename: "' + fname + '". Currently operating in ' + ('Partial' if force_partial_verification else 'Full') + ' Verification Mode') print('Distributing metadata file ' + fname + ' to ECU ' + repr(ecu_serial)) binary_data = xmlrpc_client.Binary(open(fname, 'rb').read()) #TODO: Print to be deleted print(str('%s %s' % (I_TO_PRINT, 'Returning binary_data'))) #TODO: Until here return binary_data
def __request(self, methodname, params): #Presendfunctions if methodname in ['getmeta']: if len(params) == 0: params = ('', ) if methodname in ['settext', 'appendtext']: if type(params[1]) == bytes: raise TypeError('Bytes not allowed') if six.PY2: if methodname in ['setbytes', 'appendbytes']: if type(params[1]) == unicode: raise TypeError('Unicode not allowed') if methodname in ['setbytes', 'appendbytes']: params = (params[0], xmlrpclib.Binary(params[1])) #Send func = getattr(self.proxy, methodname) try: data = func(*params) # ~ try: # ~ print(methodname, params,'-->',data) # ~ except: # ~ pass except ExpatError as err: raise errors.InvalidCharsInPath(err) except Fault as err: err = str(err) if 'fs.errors' in err: x = err.split('fs.errors.')[1].split("'")[0] errorobj = getattr(errors, x) raise errorobj(err, '') elif 'exceptions.TypeError' in err: raise TypeError(err) elif 'ExpatError' in err: raise errors.InvalidCharsInPath(err) else: # ~ print(err) raise #Postsendfunctions if methodname in ['getbytes']: data = data.data if methodname in ['getmeta']: if 'invalid_path_chars' in data: data['invalid_path_chars'] = data[ 'invalid_path_chars'].data.decode('utf-8') if six.PY2: if methodname in ['getinfo', 'getdetails']: data['basic']['name'] = data['basic']['name'].decode('utf-8') if methodname in ['listdir']: outlist = [] for entry in data: outlist.append(entry.decode('utf-8')) data = outlist if methodname in ['gettext']: #Ugly Hack to let the Tests run through try: data = data.decode('utf-8') except: pass if methodname in ['getinfo', 'getdetails']: data = info.Info(data) return data
def execute_robot_run(test_suites, dependencies, robot_args, debug=False): """ Callback that is invoked when a request to execute a robot run is made :param test_suites: Dictionary of suites to execute :type test_suites: dict :param dependencies: Dictionary of files that the test suites are dependant on :type dependencies: dict :param robot_args: Dictionary of arguments to pass to robot.run() :type robot_args: dict :param debug: Run in debug mode. This changes the logging level and does not cleanup the workspace :type debug: bool :return: Dictionary containing test results and artifacts :rtype: dict """ workspace_dir = None std_out_err = None old_cwd = None try: old_log_level = logger.level if debug: logger.setLevel(logging.DEBUG) # Save all suites & dependencies to disk workspace_dir = RobotFrameworkServer._create_workspace(test_suites, dependencies) # Change the CWD to the workspace old_cwd = os.getcwd() os.chdir(workspace_dir) sys.path.append(workspace_dir) # Execute the robot run std_out_err = StringIO() logger.debug('Beginning Robot Run.') logger.debug('Robot Run Args: ' + str(robot_args)) ret_code = run('.', stdout=std_out_err, stderr=std_out_err, outputdir=workspace_dir, name='Root', **robot_args) logger.debug('Robot Run finished') # Read the test artifacts from disk output_xml, log_html, report_html = RobotFrameworkServer._read_robot_artifacts_from_disk(workspace_dir) ret_val = {'std_out_err': xmlrpc_client.Binary(std_out_err.getvalue().encode('utf-8')), 'output_xml': xmlrpc_client.Binary(output_xml.encode('utf-8')), 'log_html': xmlrpc_client.Binary(log_html.encode('utf-8')), 'report_html': xmlrpc_client.Binary(report_html.encode('utf-8')), 'ret_code': ret_code} except Exception as e: # Log here because the RPC framework doesn't give the client a full stacktrace logging.error(e) raise finally: if old_cwd: os.chdir(old_cwd) if std_out_err: std_out_err.close() if workspace_dir and not debug: shutil.rmtree(workspace_dir) logger.debug('End of RPC function') # Revert the logger back to its original level logger.setLevel(old_log_level) return ret_val
def _checkRefPermissions(self, requester, translated_path, ref_paths, auth_params): if requester == LAUNCHPAD_ANONYMOUS: requester = None repository = removeSecurityProxy( getUtility(IGitLookup).getByHostingPath(translated_path)) if repository is None: raise faults.GitRepositoryNotFound(translated_path) try: macaroon_raw = auth_params.get("macaroon") if macaroon_raw is not None: verified = self._verifyMacaroon(macaroon_raw, repository) if not verified: # Macaroon authentication failed. Don't fall back to # the requester's permissions, since macaroons typically # have additional constraints. raise faults.Unauthorized() # Internal macaroons may only be used by internal services, # and user macaroons may only be used by real users. Forbid # potential confusion. internal = _is_issuer_internal(verified) if (requester == LAUNCHPAD_SERVICES) != internal: raise faults.Unauthorized() if internal: if not _can_internal_issuer_write(verified): raise faults.Unauthorized() # We know that the authentication parameters # specifically grant access to this repository because # we were able to verify the macaroon using the # repository as its context, so we can bypass other # checks and grant access as an anonymous repository # owner. This is only permitted for selected macaroon # issuers used by internal services. requester = GitGranteeType.REPOSITORY_OWNER elif requester == LAUNCHPAD_SERVICES: # Internal services must authenticate using a macaroon. raise faults.Unauthorized() except faults.Unauthorized: # XXX cjwatson 2019-05-09: It would be simpler to just raise # this directly, but turnip won't handle it very gracefully at # the moment. It's possible to reach this by being very unlucky # about the timing of a push. return [ (xmlrpc_client.Binary(ref_path.data), []) for ref_path in ref_paths] if all(isinstance(ref_path, xmlrpc_client.Binary) for ref_path in ref_paths): # New protocol: caller sends paths as bytes; Launchpad returns a # list of (path, permissions) tuples. (XML-RPC doesn't support # dict keys being bytes.) ref_paths = [ref_path.data for ref_path in ref_paths] return [ (xmlrpc_client.Binary(ref_path), self._renderPermissions(permissions)) for ref_path, permissions in repository.checkRefPermissions( requester, ref_paths).items() ] else: # Old protocol: caller sends paths as text; Launchpad returns a # dict of {path: permissions}. # XXX cjwatson 2018-11-21: Remove this once turnip has migrated # to the new protocol. git ref paths are not required to be # valid UTF-8. return { ref_path: self._renderPermissions(permissions) for ref_path, permissions in repository.checkRefPermissions( requester, ref_paths).items() }
def _dispatch(self, method, params): if not method in self._allowed_methods: # ~ print('Server',method,params,'-->','Unsupported') raise errors.Unsupported # ~ return func(*params) try: func = getattr(self.fs, method) params = list(params) if six.PY2: if method in ['match']: params[1] = params[1].decode('utf-8') else: params[0] = params[0].decode('utf-8') if method in ['appendtext','settext']: #Ugly Hack to let the Tests run through try: params[1] = params[1].decode('utf-8') except: pass if method in ['copy','move','copydir','movedir']: params[1] = params[1].decode('utf-8') if method in ['setbytes','appendbytes']: try: params[1] = params[1].data except: # ~ print('Server',method,params,'-->','TypeError: Need an xmlrpc.Binary object') raise TypeError('Need an xmlrpc.Binary object') if method in ['settimes']: if isinstance(params[1], xmlrpclib.DateTime): params[1] = datetime.strptime(params[1].value, "%Y%m%dT%H:%M:%S") if len(params) > 2: if isinstance(params[2], xmlrpclib.DateTime): params[2] = datetime.strptime(params[2].value, "%Y%m%dT%H:%M:%S") returndata = func(*params) if method in ['makedir',"makedirs"]: returndata = True if method in ['getinfo','getdetails']: returndata = returndata.raw if method in ['getbytes']: returndata = xmlrpclib.Binary(returndata) if method in ['getmeta']: if 'invalid_path_chars' in returndata: returndata['invalid_path_chars'] = xmlrpclib.Binary(returndata['invalid_path_chars'].encode('utf-8')) # ~ try: # ~ print('Server',method,params,'-->',returndata) # ~ except: # ~ pass return returndata except: # ~ import traceback # ~ print('############## Traceback from Server ####################') # ~ print('Server',method,params,'-->','Error') # ~ traceback.print_exc() # ~ print('############## Traceback from Server ####################') raise