def p_limit(p): r"""limit : | LIMIT INTEGER_LITERAL | LIMIT INTEGER_LITERAL COMMA INTEGER_LITERAL""" if len(p) == 1: p[0] = None elif len(p) == 3: p[0] = Limit(None, p[2]) else: p[0] = Limit(p[2], p[4])
def __init__(self, recipient=None, start=None, end=None, limit=None): if recipient is None: self._recipient = '' else: self._recipient = recipient if start is None or end is None: self._dataURL = DataURL() else: self._dataURL = DataURL(start, end) if limit is None: self._limit = Limit() else: self._limit = Limit(limit) return
def _load_from_file(self) -> None: try: with open(self.config_file, 'r') as fh: conf = yaml.safe_load(fh) except FileNotFoundError as ex: raise ConfigException( f"Config file {self.config_file} not found: {ex}") except yaml.YAMLError as ex: raise ConfigException( f"Failed to load config from {self.config_file}: {ex}") if not isinstance(conf, dict): raise ConfigException("Config must be a YAML object") self.qdir_root = conf.get("qdir_root") self.max_workers = conf.get("max_workers", self.max_workers) self.hint_origin = conf.get("hint_origin") limit_raw = conf.get("limit", {}) self.limit = Limit(memory=self._parse_mem(limit_raw.get("memory")), swap=self._parse_mem(limit_raw.get("swap")), cpu=self._parse_proc(limit_raw.get("cpu"))) if self.qdir_root is None: raise ConfigException("Field 'qdir_root' must be set") courses0 = conf.get("courses", []) if not (isinstance(courses0, list) or isinstance(courses0, dict)): raise ConfigException( "Courses must be an array or map of course objects") if isinstance(courses0, dict): courses = [] for n, c in courses0.items(): if "name" not in c: c["name"] = n courses.append(c) else: courses = courses0 for c in courses: cc = Course(c, self.qdir_root) self.courses[cc.name.lower()] = cc out = len([ x for x in [self.socket, self.socket_fd, self.port] if x is not None ]) if out == 0: self.port = 8080 if out > 1: raise ConfigException("At most one of '--socket', '--socket-fd' " "or '--port' must be used") if len(self.courses) == 0: raise ConfigException("At least one course must be set") self.postgres = conf.get("postgres", False) self.postgres_host = conf.get("postgres_host", self.postgres_host) self.postgres_user = conf.get("postgres_user", self.postgres_user)
def from_json(self, json): required = ['recipient', 'data_url', 'limit'] if not all(k in json for k in required): logging.warning(f"value missing in {required}") return False data_url = DataURL() if not data_url.from_json(json['data_url']): return False self._dataURL = data_url if not isinstance(json['recipient'], str) or not isinstance( json['limit'], int): logging.warning( "recipient should be all type<str> and limit should be type<int>" ) return False self._recipient = json['recipient'] self._limit = Limit(json['limit']) return True
def __init__(self, argv: List[str]): self.argv = argv self.config_file = "exprtest.yaml" self.socket_fd: Optional[int] = None self.socket: Optional[str] = None self.port: Optional[int] = None self.qdir_root: Optional[str] = None self.courses: Dict[str, Course] = {} self.max_workers = 4 self.hint_origin: Optional[str] = None self.limit = Limit() self._load_from_argv() self._load_from_file()
def setUp(self): self._resource1 = Resource('test_resource1', [ Limit(2, timedelta(seconds=1)), Limit(3, timedelta(seconds=1)), ]) self._resource2 = Resource('test_resource2', [ Limit(6, timedelta(seconds=1)), Limit(7, timedelta(seconds=1)), ]) self._resource3 = Resource('test_resource3', [ Limit(10, timedelta(seconds=1)), Limit(10, timedelta(seconds=1)), ])
def __init__(self, argv: List[str]) -> None: self.argv = argv self.config_file = "exprtest.yaml" self.socket_fd: Optional[int] = None self.socket: Optional[str] = None self.port: Optional[int] = None self.qdir_root: Optional[str] = None self.courses: Dict[str, Course] = {} self.max_workers = 4 self.hint_origin: Optional[str] = None self.limit = Limit() self.verbose: bool = False self.journal: bool = False self.postgres: bool = True self.postgres_host: str = "/var/run/postgresql" self.postgres_user: str = getpass.getuser() self.exprtest_stamp = _EXPRTEST_GIT_STAMP self._load_from_argv() self._load_from_file() self._setup_logging()
def limit(config): allTasks = [] for mass in config["masses"]: datacardTasks = [] haddTasks = [] tempConf = copy.deepcopy(config) for channel in list(config.keys())[1:]: for key in tempConf[channel].keys(): if type(tempConf[channel][key]) == str: tempConf[channel][key] = tempConf[channel][key].replace( "@", str(mass)) elif type(tempConf[channel][key]) == list: tempConf[channel][key] = [ i.replace("@", str(mass)) for i in tempConf[channel][key] ] tempConf[channel][ "processes"] = tempConf[channel]["backgrounds"] + [ tempConf[channel]["signal"] ] + [tempConf[channel]["data"]] treeTasks = TreeRead.configure(tempConf, channel) haddTasks.extend(HaddPlot.configure(tempConf, treeTasks, channel)) datacardTasks.extend( Datacard.configure(tempConf, channel, mass, haddTasks)) allTasks.extend(treeTasks) allTasks.extend(haddTasks + datacardTasks + Limit.configure(tempConf, mass, datacardTasks)) allTasks.extend(PlotLimit.configure(config)) return allTasks
def run(self, *args): """ """ def set_list(dict_input): for key in dict_input: dict_input[key] = list(set(dict_input[key])) return dict_input lvl1 = args[0] logger.info('Received lvl1 list: {}'.format(len(lvl1))) lvl2 = args[1] logger.info('Received lvl2 list: {}'.format(len(lvl2))) lvl3 = args[2] logger.info('Received lvl3 list: {}'.format(len(lvl3))) # calculate the limit in case of a lvl1 removed self.max_limit = Limit(self.loop_interval).current_list(lvl1) # add the loop value to the dict to parse it during the preparing # return dictionary lvl1 = self.preparing_lvl1(lvl1) lvl2, self.lvl2_loop = self.get_loop(lvl2, self.lvl2_loop) lvl2 = self.preparing_lvl2(lvl1, lvl2, self.time_lvl2, self.max_limit) lvl3 = self.preparing_lvl3(lvl1, lvl2, lvl3, self.max_limit) self.lvl2_loop = self.remove_loop(lvl2, self.lvl2_loop) return set_list(lvl1), set_list(lvl2), set_list(lvl3)
def set_limit(self, limit_min, limit_max): self.limit = Limit(limit_min, limit_max)
class SourceReader: def __init__(self, date, product, catalogue, url, index, login_data): self.date = date self.product = product self.catalogue = catalogue self.url = url self.index = index self.login_data = login_data self.is_last = False self.auth = False self.beg_time = None self.end_time = None self.row_manager = SourceRowManager(self.date) def set_beg_time(self, time): self.beg_time = time def set_end_time(self, time): self.end_time = time def set_log(self, log): self.log = log def set_limit(self, limit_min, limit_max): self.limit = Limit(limit_min, limit_max) def set_iter(self, iter_type, index): self.iter = SourceIterator(iter_type, index) self.iter_type = iter_type if self.iter_type == 'HH:MM:SS': self.hour = 0 def next_bulk(self): if self.is_last: return None data, limit = self._get_data() if data == None: return None self.first_row = data.next() lines = 0 fullBulk = False while not fullBulk: fullBulk = True result = [] #try: for cur_row in data: lines += 1 row = self._get_row( cur_row ) if row == None: self.log("ERROR! Failed to get row.") continue if row == 'ignore': continue json_row = json.dumps(row) result.append(json_row) #except: # self.log("EXCEPTION! Failed to get row. id=["+self.iter.get_str()+"]") # if lines != limit: # fullBulk = False if self.iter_type == 'HH:MM:SS': self.hour += 1 if lines < 2: if not (self.iter_type == 'HH:MM:SS' and self.hour <= 24): self.log("FINISH. id=["+self.iter.get_str()+"] limit=["+str(limit)+"]") self.is_last = True else: self.log("Bulk got. id=["+self.iter.get_str()+"] limit=["+str(limit)+"]") return result def _get_data(self): while True: url = self._get_url() try: response = requests.get(url) except KeyboardInterrupt: self.log("FINISH") return None, 0 except: self.log("GET_DATA ERROR! id=[" + self.iter.get_str() + "]") self.limit.decrease() sleep(1) continue if response.status_code != 200: self.log( "GET_DATA ERROR! code=" + str(response.status_code) + \ " text=" + response.text) self.limit.decrease() continue limit = self.limit.get() self.limit.increase() stream = StringIO.StringIO( response.text.encode('utf-8') ) return csv.reader( stream, delimiter=',' ), limit def _get_row(self, data): if len(data) <= 0: self.log("ROW ERROR! empty") return None if self.iter_type == 'id': self.iter.set(data[0]) else: self.iter.set(data[1]) row = self.row_manager.init_row() i = 0 while i < len(data): self._add_param(self.first_row[i], data[i], row) i += 1 self.row_manager.apply_rules_to_row(row) return row def _add_param(self, key, value, obj): obj[key] = value def _get_url(self): params = "date=" + self.date + \ self.iter.get_param()+"&p="+self.product+\ "&limit=" + str(self.limit.get())+"&pc="+self.catalogue if self.beg_time != None: params += "&beg_time=" + str(self.beg_time) if self.end_time != None: params += "&end_time=" + str(self.end_time) params = self._add_auth(params) url = self.url + "&" + params return url def _add_auth(self, params): if self.login_data == None: return params if self.login_data.get('login') == None: return params if self.login_data.get('password') == None: return params if self.auth == False: if self._auth() != True: return params params += '&login='******'login'] data = params.split('&') data.sort() data = ''.join(data) data += self.login_data['password'] sign = md5.new(data).hexdigest() params += '&sign=' + sign return params def _auth(self): url = self.url.split('?')[0] url += '?method=auth' url += '&login='******'login'] url += '&password='******'password'] try: response = requests.get(url) except: return False return True
def setUp(self): super().setUp() self._limit = Limit(2, timedelta(seconds=3))
def upload_data(): # check if got enough values get_json = request.get_json() print(get_json) print(type(get_json)) required = ['public_key', 'data', 'timestamp', 'op', 'signature'] if not all(k in get_json for k in required): return 'Missing values', 400 public_key = get_json['public_key'] data = get_json['data'] timestamp = get_json['timestamp'] op = get_json['op'] hash = hashlib.sha256((str(data) + str(timestamp) + str(0)).encode()).hexdigest() signature = get_json['signature'] # check if op is upload operation if op != 0: return 'op is not matched!', 400 # check if there is enough room if len(self._database) + len(get_json['data']) >= MAX_DATA_RANGE: return 'no enough storage', 400 # check if timestamp is not expired if timestamp + 600 < time.time(): return 'Request expired', 400 # check if signature is matched print(signature) print(type(signature)) if not verify_signature(public_key, hash, eval(signature)): return 'Signature unmatched', 400 # everything is fine then store data into database if type(data) is 'list': self._database += data else: self._database.append(data) # generate a new authorization input = Input(0, 0, 0) if type(data) is 'list': data_url = DataURL(len(self._database) - len(data), len(self._database)) else: data_url = DataURL(len(self._database) - 1, len(self._database)) output = Output(recipient=public_key, data_url=data_url, limit=Limit(7)) authorization = Authorization(inputs=[input], outputs=[output], duration=Duration(), timestamp=time.time()) # sign this authorization input.add_signature(self._account.sign_message(authorization.calc_hash())) # store this authorization and broadcast it auth_number = self.add_authorization(authorization) print(auth_number) # return the position of authorization to client response = { 'block_number': self._blockchain.get_height(), 'auth_number': auth_number, 'output_number': 0 } return jsonify(response), 201
def set_limit(self, limit_min, limit_max): self.limit = Limit(limit_min, limit_max)
class SourceReader: def __init__(self, date, product, catalogue, url, index, login_data): self.date = date self.product = product self.catalogue = catalogue self.url = url self.index = index self.login_data = login_data self.is_last = False self.auth = False self.beg_time = None self.end_time = None self.row_manager = SourceRowManager(self.date) def set_beg_time(self, time): self.beg_time = time def set_end_time(self, time): self.end_time = time def set_log(self, log): self.log = log def set_limit(self, limit_min, limit_max): self.limit = Limit(limit_min, limit_max) def set_iter(self, iter_type, index): self.iter = SourceIterator(iter_type, index) self.iter_type = iter_type if self.iter_type == 'HH:MM:SS': self.hour = 0 def next_bulk(self): if self.is_last: return None data, limit = self._get_data() if data == None: return None self.first_row = data.next() lines = 0 fullBulk = False while not fullBulk: fullBulk = True result = [] #try: for cur_row in data: lines += 1 row = self._get_row(cur_row) if row == None: self.log("ERROR! Failed to get row.") continue if row == 'ignore': continue json_row = json.dumps(row) result.append(json_row) #except: # self.log("EXCEPTION! Failed to get row. id=["+self.iter.get_str()+"]") # if lines != limit: # fullBulk = False if self.iter_type == 'HH:MM:SS': self.hour += 1 if lines < 2: if not (self.iter_type == 'HH:MM:SS' and self.hour <= 24): self.log("FINISH. id=[" + self.iter.get_str() + "] limit=[" + str(limit) + "]") self.is_last = True else: self.log("Bulk got. id=[" + self.iter.get_str() + "] limit=[" + str(limit) + "]") return result def _get_data(self): while True: url = self._get_url() try: response = requests.get(url) except KeyboardInterrupt: self.log("FINISH") return None, 0 except: self.log("GET_DATA ERROR! id=[" + self.iter.get_str() + "]") self.limit.decrease() sleep(1) continue if response.status_code != 200: self.log( "GET_DATA ERROR! code=" + str(response.status_code) + \ " text=" + response.text) self.limit.decrease() continue limit = self.limit.get() self.limit.increase() stream = StringIO.StringIO(response.text.encode('utf-8')) return csv.reader(stream, delimiter=','), limit def _get_row(self, data): if len(data) <= 0: self.log("ROW ERROR! empty") return None if self.iter_type == 'id': self.iter.set(data[0]) else: self.iter.set(data[1]) row = self.row_manager.init_row() i = 0 while i < len(data): self._add_param(self.first_row[i], data[i], row) i += 1 self.row_manager.apply_rules_to_row(row) return row def _add_param(self, key, value, obj): obj[key] = value def _get_url(self): params = "date=" + self.date + \ self.iter.get_param()+"&p="+self.product+\ "&limit=" + str(self.limit.get())+"&pc="+self.catalogue if self.beg_time != None: params += "&beg_time=" + str(self.beg_time) if self.end_time != None: params += "&end_time=" + str(self.end_time) params = self._add_auth(params) url = self.url + "&" + params return url def _add_auth(self, params): if self.login_data == None: return params if self.login_data.get('login') == None: return params if self.login_data.get('password') == None: return params if self.auth == False: if self._auth() != True: return params params += '&login='******'login'] data = params.split('&') data.sort() data = ''.join(data) data += self.login_data['password'] sign = md5.new(data).hexdigest() params += '&sign=' + sign return params def _auth(self): url = self.url.split('?')[0] url += '?method=auth' url += '&login='******'login'] url += '&password='******'password'] try: response = requests.get(url) except: return False return True
class Output: def __init__(self, recipient=None, start=None, end=None, limit=None): if recipient is None: self._recipient = '' else: self._recipient = recipient if start is None or end is None: self._dataURL = DataURL() else: self._dataURL = DataURL(start, end) if limit is None: self._limit = Limit() else: self._limit = Limit(limit) return def set_recipient(self, recipient): self._recipient = recipient return def get_recipient(self): return self._recipient def set_data_url(self, data_url): self._dataURL = data_url return def get_data_url(self): return self._dataURL def data_url_contains(self, start, end): return self._dataURL.contains(start, end) def data_url_belongs(self, start, end): return self._dataURL.belongs(start, end) def set_limit(self, limit): self._limit.set(limit) return def get_limit(self): return self._limit def valid_limit(self, limit): return self._limit.valid(limit) def valid(self, givens): # print(givens) for given in givens: if self.data_url_belongs(given[0], given[1]) and self.valid_limit( given[2]): return True logging.warning( 'this output does not match any given data_url or limit') return False def to_json(self): json = { 'recipient': self._recipient, 'data_url': self._dataURL.to_json(), 'limit': self._limit.value() } return json def from_json(self, json): required = ['recipient', 'data_url', 'limit'] if not all(k in json for k in required): logging.warning(f"value missing in {required}") return False data_url = DataURL() if not data_url.from_json(json['data_url']): return False self._dataURL = data_url if not isinstance(json['recipient'], str) or not isinstance( json['limit'], int): logging.warning( "recipient should be all type<str> and limit should be type<int>" ) return False self._recipient = json['recipient'] self._limit = Limit(json['limit']) return True def __str__(self): return str(self._recipient) + str(self._dataURL) + str(self._limit)