Ejemplo n.º 1
0
 def save(self):
     predicsis.log('Updating: ' + self.__class__.res_name() + '..', 1)
     post_data = self.__class__.parse_post_data(self.to_update)
     json_data = APIClient.request('patch', self.__class__.res_url() + '/' + self.id, post_data)
     j = json_data[self.__class__.res_name()]
     try:
         jid = j['job_ids'][0]
         status = 'pending'
         job = Job.retrieve(jid)
         status = job.status
         while ((status != 'completed') and (status != 'failed')):
             job = Job.retrieve(jid)
             status = job.status
             if status == 'failed':
                 raise PredicSisError("Job failed! (job_id: " + job.id + ")")
             #time.sleep(1)
         json_data = APIClient.request('get', self.res_url() + '/' + self.id)
         obj = json_data[self.res_name()]
         for k, v in obj.iteritems():
             if isinstance(v, dict):
                 setattr(self, k, APIResource(v))
             else:
                 setattr(self, k, v)
     except KeyError:
         json_data = APIClient.request('get', self.res_url() + '/' + self.id)
         obj = json_data[self.res_name()]
         for k, v in obj.iteritems():
             if isinstance(v, dict):
                 setattr(self, k, APIResource(v))
             else:
                 setattr(self, k, v)
     except IndexError:
         raise PredicSisError("Job launching failed. Report this bug to [email protected]");
 def request(cls, method, resource, post_data=None):
     predicsis.log(method.upper() + ' ' + predicsis.api_url + resource+ ' [' + str(post_data) + ']', 2)
     headers = {'Accept': 'application/json'}
     if (method == 'post') or (method == 'patch'):
         headers['Content-Type'] = 'application/json'
     headers['Authorization'] = 'Bearer ' + predicsis.api_token;
     content, code, json = cls.request_full(method, predicsis.api_url + resource, headers, post_data)
     return cls._interpret_response(content, code, json)
Ejemplo n.º 3
0
 def create(cls, **data):
     if validate('c', cls.__name__, data) < 0:
         raise PredicSisError("Validation failed!")
     response = Model.retrieve(data.get('model_id'))
     prs_id = response.preparation_rules_set_id
     response = PreparationRules.retrieve(prs_id)
     var_id = response.variable_id
     Variable.dictionary_id = data.get('dictionary_id')
     response = Variable.retrieve(var_id)
     modalities_set_id = response.modalities_set_ids[0]
     response = ModalitiesSet.retrieve(modalities_set_id)
     modalities = response.modalities
     predicsis.log('Preparing data..', 1)
     dataset_id = -1
     file_name = "./tmp.dat"
     try:
         if not exists(data.get('data')):
             predicsis.log('The file doesn\'t exist: ' + data.get('data') + '. Passing this value as a content.', 0)
         open(data.get('data'),'rb')
         if data.get('header') == None or data.get('separator') == None:
             if not (data.get('header') == None and data.get('separator') == None):
                 predicsis.log('Either both separator and header should be set, or both should be left unset. The set parameter is skipped.', 0)
             dataset_id = Dataset.create(file=data.get('data'), name = data.get('name')).id
         else:
             dataset_id = Dataset.create(file=data.get('data'), header=data.get('header'), separator=data.get('separator'), name = data.get('name')).id
     except IOError:
         f = open(file_name,'w')
         f.write(data.get('data'))
         f.close()
         if data.get('header') == None or data.get('separator') == None:
             if not (data.get('header') == None and data.get('separator') == None):
                 predicsis.log('Either both separator and header should be set, or both should be left unset. The set parameter is skipped.', 0)
             dataset_id = Dataset.create(file=file_name, name = data.get('name')).id
         else:
             dataset_id = Dataset.create(file=file_name, header=data.get('header'), separator=data.get('separator'), name = data.get('name')).id
     if dataset_id == -1:
         raise PredicSisError("Error creating your test dataset")
     scoresets = []
     for modality in modalities:
         if data.get('header') == None or data.get('separator') == None:
             if not (data.get('header') == None and data.get('separator') == None):
                 predicsis.log('Either both separator and header should be set, or both should be left unset. The set parameter is skipped.', 0)
             dataset = DatasetAPI.create(name=data.get('name'), classifier_id=data.get('model_id'), dataset_id=dataset_id, modalities_set_id=modalities_set_id, main_modality=modality, data_file = { "filename": data.get('file_name')})
         else:
             dataset = DatasetAPI.create(name=data.get('name'), header=str(data.get('header')).lower(), separator=data.get('separator').encode('string_escape'), classifier_id=data.get('model_id'), dataset_id=dataset_id, modalities_set_id=modalities_set_id, main_modality=modality, data_file = { "filename": data.get('file_name')})
         scoreset = cls(json.loads(str(dataset)))
         scoresets.append(scoreset)
     return scoresets
Ejemplo n.º 4
0
    def create(cls, **data):
        if validate('c', cls.__name__, data) < 0:
            raise PredicSisError("Validation failed!")
        credentials = Credentials.retrieve('s3')
        payload = {
            'Content-Type':'multipart/form-data',
            'success_action_status':'201',
            'acl':'private',
            'policy':credentials.policy,
            'AWSAccessKeyId':credentials.aws_access_key_id,
            'signature':credentials.signature,
            'key':credentials.key
        }
        predicsis.log('Uploading a file..', 1)
        if not exists(data.get('file')):
			raise PredicSisError('The file doesn\'t exist: ' + data.get('file') + '.')
        files = {'file': open(data.get('file'),'rb')}
        response = APIClient.request_full(method='post', url=credentials.s3_endpoint, headers=[],post_data=payload, files=files)
        if not response[1] == 201:
            raise PredicSisError('Upload failed by Amazon - retry.')
        xmlResponse = minidom.parseString(response[0])
        keyList = xmlResponse.getElementsByTagName('Key')
        predicsis.log('Creating: dataset..', 1)
        source = Source.create(name=data.get('file'), key=str(keyList[0].firstChild.data))
        sid = str(source.id)
        if data.get('header') == None or data.get('separator') == None:
            if not (data.get('header') == None and data.get('separator') == None):
                predicsis.log('Either both separator and header should be set, or both should be left unset. The set parameter is skipped.', 0)
            dapi = DatasetAPI.create(name=data.get('name'), source_ids=[sid])
        else:
            dapi = DatasetAPI.create(name=data.get('name'), header=str(data.get('header')).lower(), separator=data.get('separator').encode('string_escape'), source_ids=[sid])
        for i in range(0,len(dapi.preview)):
            dapi.preview[i] = '...not available in the SDK...'#dapi.preview[i].replace('"','*')#
        return cls(json.loads(str(dapi)))
Ejemplo n.º 5
0
 def create(cls, **data):
     if validate('c', cls.__name__, data) < 0:
         raise PredicSisError("Validation failed!")
     predicsis.log('Creating: ' + cls.res_name() + '..', 1)
     post_data = cls.parse_post_data(data)
     json_data = APIClient.request('post', cls.res_url(), post_data)
     j = json_data[cls.res_name()]
     try:
         jid = j['job_ids'][0]
         status = 'pending'
         job = Job.retrieve(jid)
         status = job.status
         while ((status != 'completed') and (status != 'failed')):
             job = Job.retrieve(jid)
             status = job.status
             if status == 'failed':
                 raise PredicSisError("Job failed! (job_id: " + job.id + ")")
             #time.sleep(1)
         return cls.retrieve(j['id'])
     except KeyError:
         return cls(j)
     except IndexError:
         raise PredicSisError("Job launching failed. Report this bug to [email protected]");
 def request_full(cls, method, url, headers, post_data=None, files=None):
     kwargs = {}
     if verify_ssl_certs:
         kwargs['verify'] = ssl_certs_path
     else:
         kwargs['verify'] = False
         try:
             requests.packages.urllib3.disable_warnings()
         except AttributeError:
             predicsis.log('Impossible to shut down the SSL-related warnings - check the version of python and/or requests package.', 0)
     try:
         try:
             result = requests.request(method, url, headers=headers, data=post_data, files=files, timeout=80, **kwargs)
         except:
             try:
                 result = requests.request(method, url, headers=headers, data=str(post_data), files=files, timeout=80, **kwargs)
             except TypeError, e:
                 raise TypeError('Your "requests" library may be out of date. Error was: %s' % (e,))
         content = result.content
         status_code = result.status_code
         if files == None and not method=='delete':
             jsonn = result.json()
         else:
             jsonn = result
Ejemplo n.º 7
0
 def create(cls, **data):
     if validate('c', cls.__name__, data) < 0:
         raise PredicSisError("Validation failed!")
     predicsis.log('Retrieving: variables..', 1)
     target_id = -1
     unused_ids = []
     dico = Dictionary.retrieve(data.get('dictionary_id'))
     dataset_id = dico.dataset_id
     Variable.dic_id = data.get('dictionary_id')
     variables = Variable.retrieve_all()
     i = 1
     for var in variables:
         if type(data.get('target_var')).__name__ == "str":
             if var.name == data.get('target_var'):
                 target_id = var.id
                 if not var.type == 'categorical':
                     predicsis.log("Your variable is not detected as categorical - changing it manually.",0)
                     var.update(type = 'categorical')
                     var.save()
         elif type(data.get('target_var')).__name__ == "int":
             if i == data.get('target_var'):
                 target_id = var.id
                 if not var.type == 'categorical':
                     predicsis.log("Your variable is not detected as categorical - changing it manually.",0)
                     var.update(type = 'categorical')
                     var.save()
         if not data.get('unused_vars') == None:
             if var.name in data.get('unused_vars'):
                 var.update(use = False)
                 var.save()
             elif i in data.get('unused_vars'):
                 var.update(use = False)
                 var.save()
         i+=1
     if target_id == -1:
         raise PredicSisError("Your target variable doesn't exist in the dataset.")
     predicsis.log('Creating: target..', 1)
     modal = ModalitiesSet.create(variable_id = target_id, dataset_id = dataset_id)
     return cls(json.loads(str(modal)))
         try:
             result = requests.request(method, url, headers=headers, data=post_data, files=files, timeout=80, **kwargs)
         except:
             try:
                 result = requests.request(method, url, headers=headers, data=str(post_data), files=files, timeout=80, **kwargs)
             except TypeError, e:
                 raise TypeError('Your "requests" library may be out of date. Error was: %s' % (e,))
         content = result.content
         status_code = result.status_code
         if files == None and not method=='delete':
             jsonn = result.json()
         else:
             jsonn = result
     except Exception, e:
         cls._handle_request_error(e)
     predicsis.log('return status: ' + str(status_code), 2)
     if files==None and not method=='delete':
         predicsis.log(json.dumps(jsonn, indent=4), 3)
     elif not method=='delete':
         xmlResponse = minidom.parseString(result.content)
         predicsis.log(xmlResponse.toprettyxml(indent="\t"), 3)
     return content, status_code, jsonn
 
 @classmethod
 def request_direct(cls, url):
     return requests.get(url)
     
 @classmethod
 def _interpret_response(cls, content, code, json):
     if not (200 <= code < 300):
         cls._handle_api_error(content, code, json)
Ejemplo n.º 9
0
 def retrieve_all(cls):
     predicsis.log('Retrieving all: ' + cls.res_name() + '..', 1)
     json_data = APIClient.request('get', cls.res_url())
     j = json_data[cls.res_url().split('/')[-1]]
     return [cls(i) for i in j]
Ejemplo n.º 10
0
def validate(act, obj, data):
    cmandatory = {
        'dataset' : ['file', 'name'],
        'dictionary' : ['name'],
        'target' : ['target_var', 'dictionary_id'],
        'model' : ['variable_id', 'dataset_id'],
        'scoreset' : ['name', 'model_id', 'dictionary_id', 'data', 'file_name'],
        'report1' : ['type', 'dictionary_id', 'dataset_id'],
        'report2' : ['type', 'dictionary_id', 'dataset_id', 'variable_id'],
        'report3' : ['type', 'dictionary_id', 'dataset_id', 'model_id', 'main_modality', 'variable_id']
    }
    coptional = {
        'dataset' : ['header', 'separator', 'file_name'],
        'dictionary' : ['description', 'dataset_id'],
        'target' : ['unused_vars'],
        'model' : ['name'],
        'scoreset' : ['header', 'separator'],
        'report1' : ['title'],
        'report2' : ['title'],
        'report3' : ['title']
    }
    uoptional = {
        'dataset' : ['name', 'header', 'separator'],
        'dictionary' : ['name', 'description'],
        'scoreset' : ['name', 'header', 'separator'],
        'report' : ['title']
    }
    if not obj.lower() in coptional.keys() + uoptional.keys():
        predicsis.log('Unvalidated object [' + obj + ']', 1)
        return 0
    if act == 'c':
        predicsis.log('Parameters: mandatory'+ str(cmandatory.get(obj.lower())) + ', optional' + str(coptional.get(obj.lower())) + ', passed' + str(data.keys()), 3)
        if obj == 'Report':
            if data.get('type') == None:
                predicsis.log('Missing parameter to create [Report]: type')
                return -1
            if data.get('type') == 'univariate_unsupervised':
                obj = 'Report1'
            elif data.get('type') == 'univariate_supervised':
                obj = 'Report2'
            elif data.get('type') == 'classifier_evaluation':
                obj = 'Report3'
        if not all(x in data.keys() for x in cmandatory.get(obj.lower())):
            predicsis.log('Missing parameters to create [' + obj + ']: ' + str([item for item in cmandatory.get(obj.lower()) if item not in data.keys()]), -1)
            return -1
    if act == 'c':
        list = [item for item in data.keys() if item not in (coptional.get(obj.lower()) + cmandatory.get(obj.lower()))]
        if len(list) > 0:
            predicsis.log('Unnecessary parameters to create [' + obj + ']: ' + str(list), 0)
            return 0
    if act == 'u':
        predicsis.log('Parameters: optional' + str(uoptional.get(obj.lower())) + ', passed' + str(data.keys()), 3)
        list = [item for item in data.keys() if item not in uoptional.get(obj.lower())]
        if len(list) > 0:
            predicsis.log('Unnecessary parameters to update [' + obj + ']: ' + str(list), 0)
            return 0
    return 1
Ejemplo n.º 11
0
 def retrieve(cls, id):
     predicsis.log('Retrieving: ' + cls.res_name() + '..', 1)
     json_data = APIClient.request('get', cls.res_url() + '/' + id)
     j = json_data[cls.res_name()]
     return cls(j)
Ejemplo n.º 12
0
 def delete(self):
     predicsis.log('Deleting: ' + self.__class__.res_name() + '..', 1)
     APIClient.request('delete', self.__class__.res_url() + '/' + self.id)
     for id in self.source_ids:
         APIClient.request('delete', 'sources/' + id)
Ejemplo n.º 13
0
 def delete(self):
     predicsis.log('Deleting: ' + self.__class__.res_name() + '..', 1)
     APIClient.request('delete', self.__class__.res_url() + '/' + self.id)