def pull_request(app, pr_title, branch, base="staging", owner='frappe'): global config pr_body = config.get("pr_body") in_test = config.get("in_test") git_in_test = config.get("git_in_test") github_username = config.get("github_username") github_password = config.get("github_password") url = 'https://api.github.com/repos/{0}/{1}/pulls'.format(owner, app) args = { "body": pr_body, "title": pr_title, "head": "{github_username}:{branch}".format(github_username=github_username, branch=branch), "base": base } if (not in_test) and (not git_in_test): r = requests.post(url, auth=HTTPBasicAuth(github_username, github_password), data=json.dumps(args)) r.raise_for_status() print "created pull request for {0}".format(app)
def GetROIBaselineModel(learningRateName='LEARNING_RATE', stepCountName='NB_STEPS', batchSizeName='BATCH_SIZE', keepProbName='KEEP_PROB', optimizer='ADAM', firstHiddenLayerUnits=96, secondHiddenLayerUnits=0): ############ DEFINE PLACEHOLDERS, LOSS ############ predictionLayer = baselineROICNN( matricesPL, trainingPL, keepProbability=get('TRAIN.ROI_BASELINE.%s' % keepProbName), firstHiddenLayerUnits=firstHiddenLayerUnits, secondHiddenLayerUnits=secondHiddenLayerUnits) lossFunction = tf.losses.mean_squared_error(labels=labelsPL, predictions=predictionLayer) ############ DEFINE OPTIMIZER ############ if optimizer == 'ADAM': trainOperation = AdamOptimizer( lossFunction, get('TRAIN.ROI_BASELINE.%s' % learningRateName)) elif optimizer == 'GRAD_DECAY': trainOperation = ScheduledGradOptimizer( lossFunction, baseLearningRate=get('TRAIN.ROI_BASELINE.%s' % learningRateName)) ############ DEFINE LEARNING PARAMETERS ############ stepCount = get('TRAIN.ROI_BASELINE.%s' % stepCountName) batchSize = get('TRAIN.ROI_BASELINE.%s' % batchSizeName) return predictionLayer, lossFunction, trainOperation, stepCount, batchSize
def push(app, repo_path, branch, commit_msg, commit=True): from utils import config in_test = config.get("in_test") git_in_test = config.get("git_in_test") if not all([repo_path, branch, commit_msg]): raise Exception("Invalid arguments") if ("{app}" in commit_msg) and ("{date}" in commit_msg): commit_msg = commit_msg.format(app=app, date=now_date()) repo = Repo(repo_path) if commit: repo.git.add('--all') repo.index.commit(commit_msg) args = [ '{branch}:{branch}'.format(branch=branch), ] # push the changes if (not in_test) and (not git_in_test): repo.git.push('origin', *args) print "pushed the changes to repo with commit message\n{0}".format( commit_msg)
def isAlertTime(): alertFrom = datetime.strptime(config.get("alerttimewindow", "fromtime"), "%H:%M").time() alertTill = datetime.strptime(config.get("alerttimewindow", "totime"), "%H:%M").time() now = datetime.now().time() return now >= alertFrom and now < alertTill
def load_config(self): config.load() adsrconf = config.get('adsr') if adsrconf: for k, v in adsrconf.iteritems(): setattr(self.adsrList, k, v) self.adsrw.adsr = self.adsrList sounds = config.get('sounds') if sounds: for snd in sounds['files']: self.add_sound(str(snd)) globs = config.get('globals') if globs: self.invert_control = globs['invert_control'] self.trigger_shadow = globs['trigger_shadow'] if self.invert_control: self.invert.setCheckState(QtCore.Qt.Checked) if self.trigger_shadow: self.invert_slope.setCheckState(QtCore.Qt.Checked) self.sldVolumen.setValue(globs['default_volume']) for x in xrange(globs['sensor_count']): self.add_sensor()
def getAlertMessage(read, eTrend, bIsConssitant, wind_change, change_time): str_verb = strings_dict[eTrend.name]["verb"] str_trend = strings_dict[eTrend.name]["label"] + ( " " + strings_dict[bIsConssitant]["label"] if eTrend != WindTrend.STEADY else "") str_sub_message = str.format( "{verb} בכ {change}kn ", verb=str_verb, change=wind_change) if eTrend != WindTrend.STEADY else "" str_meaasge = str.format("{beach} - {trend}: {current}kn - {gust}kn {direction_name}-{direction}\n" + \ "{sub_message}ב {change_time} דקות האחרונות\n" + \ "זמן קריאה: {read_date} {read_time}\n" + \ "מקור [{source}]({source_url})", \ beach=config.get(read[consts.WINDREADSFIELDS.INFO_SOURCE_NAME], "beachName"), \ current=read[consts.WINDREADSFIELDS.WIND_AVG], \ gust=read[consts.WINDREADSFIELDS.WIND_GUST], \ direction_name=read[consts.WINDREADSFIELDS.WIND_DIR_NAME], \ direction=read[consts.WINDREADSFIELDS.WIND_DIR], \ trend=str_trend, \ sub_message=str_sub_message, \ verb=str_verb, \ change=wind_change, change_time=change_time, \ read_date=read[consts.WINDREADSFIELDS.INFO_DATE], \ read_time=read[consts.WINDREADSFIELDS.INFO_TIME], \ source=config.get(read[consts.WINDREADSFIELDS.INFO_SOURCE_NAME], "frindlyName"), \ source_url=read[consts.WINDREADSFIELDS.INFO_SOURCE_URL]) return str_meaasge
def GetBaselineLSTMModel(timecoursePL, learningRateName='LEARNING_RATE', stepCountName='NB_STEPS', batchSizeName='BATCH_SIZE', optimizer='ADAM'): ############ DEFINE PLACEHOLDERS, LOSS ############ predictionLayer = baselineLSTM(timecoursePL=timecoursePL) lossFunction = tf.losses.mean_squared_error(labels=labelsPL, predictions=predictionLayer) ############ DEFINE OPTIMIZER ############ if optimizer == 'ADAM': trainOperation = AdamOptimizer(lossFunction, get('TRAIN.LSTM_BASELINE.%s' % learningRateName), clipGrads=True) elif optimizer == 'GRAD_DECAY': trainOperation = ScheduledGradOptimizer( lossFunction, baseLearningRate=get('TRAIN.LSTM_BASELINE.%s' % learningRateName)) ############ DEFINE LEARNING PARAMETERS ############ stepCount = get('TRAIN.LSTM_BASELINE.%s' % stepCountName) batchSize = get('TRAIN.LSTM_BASELINE.%s' % batchSizeName) return predictionLayer, lossFunction, trainOperation, stepCount, batchSize
def getValidation(self,numbers,type): im_array = np.zeros(0) first=1 if(type == "images"): for num in numbers: strFilename = get('DATA.ImageDirectoryTrain')+ str(int(num)) + ".png" encodedImage = scipy.misc.imread(strFilename) if first == 1: first = 0 im_array = encodedImage[np.newaxis,...] else: im_array = np.concatenate((im_array,encodedImage[np.newaxis,...]),axis=0) elif(type == "masks"): for num in numbers: strFilename = get('DATA.ImageDirectoryTrainMask') + str(int(num)) + ".png" encodedImage = scipy.misc.imread(strFilename) if first == 1: first = 0 im_array = encodedImage[np.newaxis,...] else: im_array = np.concatenate((im_array,encodedImage[np.newaxis,...]),axis=0) elif(type == "normals"): for num in numbers: strFilename = get('DATA.ImageDirectoryTrainOutput') + str(int(num)) + ".png" encodedImage = scipy.misc.imread(strFilename) if first == 1: first = 0 im_array = encodedImage[np.newaxis,...] else: im_array = np.concatenate((im_array,encodedImage[np.newaxis,...]),axis=0) return im_array
def getWinds(): """ Downloads the page where the list of mathematicians is found and returns a list of strings, one per mathematician """ for source in [ consts.SOURCEREAD.PRIGAL, consts.SOURCEREAD.EILAT_METEO_TECH, consts.SOURCEREAD.DOR_NACHSHOLIM, consts.SOURCEREAD.SURFO ]: url = config.get(source, "readsURL") + "?t=" + str( datetime.datetime.now().strftime("%d/%m/%Y-%H:%M:%S")) windUnits = config.get(source, "windUnits") response = simple_get(url) if response is not None: try: html = BeautifulSoup(response, 'html.parser') info = windInfo(source, url, windUnits) scrapWebData(html, info) jsonInfo = info.toJSON() firedata.writeWindReads(jsonInfo) res = firedata.readWindReads(source) wind_tracker.sense_for_wind_change(res) except Exception as e: log_error('Exception during response handling: {0}'.format( str(e))) else: # Raise an exception if we failed to get any data from the url # raise Exception('Error retrieving contents at {}'.format(url)) log_error('Error retrieving contents at {}'.format(url))
def restoreAttention(): additionalArgs = [{ 'flag': '--strideSize', 'help': 'The stride to chunk MRI images into. Typical values are 10, 15, 20, 30, 40, 60.', 'action': 'store', 'type': int, 'dest': 'strideSize', 'required': True }, { 'flag': '--type', 'help': 'One of: traditional, reverse', 'action': 'store', 'type': str, 'dest': 'type', 'required': True }, { 'flag': '--attention', 'help': 'One of: 0, 1, 2, 3', 'action': 'store', 'type': int, 'dest': 'attention', 'required': True }] ParseArgs('Run 3D CNN over structural MRI volumes', additionalArgs=additionalArgs, useDefaults=False) if GlobalOpts.strideSize <= 0: GlobalOpts.strideSize = None GlobalOpts.imageBaseString = get('DATA.STRUCTURAL.DOWNSAMPLE_PATH') GlobalOpts.trainFiles = np.load(get('DATA.TRAIN_LIST')).tolist() GlobalOpts.imageBatchDims = (-1, 61, 73, 61, 1) GlobalOpts.trainBatchSize = 4 GlobalOpts.augment = 'none' GlobalOpts.name = 'attention{}_{}_stride{}'.format(GlobalOpts.attention, GlobalOpts.type, GlobalOpts.strideSize) modelTrainer = ModelTrainer() GlobalOpts.checkpointDir = '{}{}/'.format('../checkpoints/attention_comp/', GlobalOpts.name) imagesPL, labelsPL = StructuralPlaceholders(GlobalOpts.imageBatchDims) trainingPL = TrainingPlaceholder() if GlobalOpts.type == 'traditional': GlobalOpts.convLayers = [8, 16, 32, 64] elif GlobalOpts.type == 'reverse': GlobalOpts.convLayers = [64, 32, 16, 8] GlobalOpts.randomAttentionStarter = False if GlobalOpts.attention == 0: GlobalOpts.attentionMapBools = [True, False, False, False] elif GlobalOpts.attention == 1: GlobalOpts.attentionMapBools = [False, True, False, False] elif GlobalOpts.attention == 2: GlobalOpts.attentionMapBools = [True, True, False, False] elif GlobalOpts.attention == 3: GlobalOpts.attentionMapBools = [False, False, False, False] GlobalOpts.randomAttentionStarter = True saveFilters(GlobalOpts, imagesPL, trainingPL)
def train_cnn(input_layer, prediction_layer, loss_func, optimizer, faces): ''' Train CNN ''' try: for batch_index in range(get('TRAIN.CNN.NB_STEPS')): report_training_progress(batch_index, input_layer, loss_func, faces) batch_images, batch_masks, batch_normals = faces.train.next_batch(get('TRAIN.CNN.BATCH_SIZE')) optimizer.run(feed_dict={input_layer: batch_images, masks: batch_masks, true_normals: batch_normals}) except KeyboardInterrupt: print('OK, I will stop training even though I am not finished.')
def update_benches(): """ update the both frappe-bench and release-bench """ from utils import config update_bench = config.get("update_bench") release_bench = config.get("release_bench") for bench in [update_bench, release_bench]: exec_cmd(bench, ['bench update --no-backup --reset'])
def MatrixPlaceholders(): """ Returns input and output placeholders for the connectivity matrices in the data file. """ matricesPL = tf.placeholder(dtype=tf.float32, shape=(None, get('DATA.MATRICES.DIMENSION'), get('DATA.MATRICES.DIMENSION'), 1), name='matricesPL') labelsPL = tf.placeholder(dtype=tf.float32, shape=(None, 1), name='labelsPL') return (matricesPL, labelsPL)
def readDateTime(self, Value): readDateTime = self.getDateTime(Value) if readDateTime is not None: self._infoDateTime = readDateTime dFormat = config.get( self.infoSourceName, "dateFormat") if config.get( self.infoSourceName, "dateFormat") else '%d/%m/%y' tFormat = config.get( self.infoSourceName, "timeFormat") if config.get( self.infoSourceName, "timeFormat") else '%H:%M' self.infoDate = readDateTime.strftime(dFormat) self.infoTime = readDateTime.strftime(tFormat)
def SlicePlaceholders(): """ Returns input and output placeholders for the slice images in the /data directory. """ slicesPL = tf.placeholder(dtype=tf.float32, shape=(None, get('DATA.SLICES.DIMENSION'), get('DATA.SLICES.DIMENSION'), 1), name='slicesPL') labelsPL = tf.placeholder(dtype=tf.float32, shape=(None, 1), name='labelsPL') return (slicesPL, labelsPL)
def __init__(self): set_section("db") self.__username = get('username') self.__password = get('passwd') self.__db = get('dbname') self.__port = int(get('port')) self.__host = get('host') self.__charset = 'utf8' self.conn = None self.cursor = None self.times = 0 self.connect_db()
def getDateTime(val): ''' This function will check the type of the val if its a string type then uses regex to split the time from the string characters ie if val is 01/02 09:20 then will return 09:20 ''' val.strip dateTimeFormat = config.get( consts.SOURCEREAD.EILAT_METEO_TECH, "dateFormat") + " " + config.get( consts.SOURCEREAD.EILAT_METEO_TECH, "timeFormat") readDateTime = datetime.datetime.strptime(val.strip(), dateTimeFormat) if readDateTime.year == 1900: readDateTime = readDateTime.replace(year=datetime.datetime.now().year) return readDateTime
def TimecoursePlaceholders(): """ Returns input and output placeholders for the timecourses in the /data directory. """ timecoursePL = tf.placeholder(tf.float32, shape=(None, get('DATA.TIMECOURSES.SEQ_LENGTH'), get('DATA.TIMECOURSES.SEQ_WIDTH')), name='timecoursePL') labelsPL = tf.placeholder(dtype=tf.float32, shape=(None, 1), name='labelsPL') return (timecoursePL, labelsPL)
def build_docs(apps, _pull=False): """ build frappe app's documentation """ from utils import config release_bench = config.get("release_bench") release_bench_site = config.get("release_bench_site") commit_msg = config.get("docs_commit_msg") target_app_mapper = config.get("docs_target_apps") base_branches = config.get("base_branch_mapper") or {} branch = "docs-{0}".format(now_date(format='%Y.%m.%d')) for app in apps: try: apps_path = os.path.join(release_bench, 'apps', app) if not apps_path: print "app is not installed" target = target_app_mapper.get(app, {}) target_app = target.get("app", None) owner = target.get("owner", None) if not target_app or not owner: print "target app mapping not available" target_app_path = os.path.join(release_bench, 'apps', target_app) if not target_app_path: print "app is not installed" if _pull: pull(apps_path, "upstream", base_branches.get(app, "develop")) pull(target_app_path, "upstream", base_branches.get(target_app, "master")) checkout(target_app_path, branch, create_new=True) # build docs exec_cmd(release_bench, \ ['bench --site {0} build-docs --target {1} {2}'.format(release_bench_site, target_app, app)]) push(app, target_app_path, branch, commit_msg, commit=True) pull_request(target_app, commit_msg, branch, base=base_branches.get(target_app, "master"), owner=owner) checkout(target_app_path, base_branches.get(target_app, "master"), delete_branch_after_checkout=False, delete_branch=branch) except Exception as e: print e
def save_config(self): adsrconf = config.get('adsr') for param in adsr_params.keys(): adsrconf[param] = getattr(self.adsrList, param) sounds = config.get('sounds') sounds['files'] = self.sounds[:] globs = config.get('globals') globs['invert_control'] = self.invert_control globs['trigger_shadow'] = self.trigger_shadow globs['default_volume'] = self.sldVolumen.value() globs['sensor_count'] = len(self.players) config.save()
def train_autoencoder(input_layer, loss_func, optimizer, faces): ''' Train autoencoder. ''' # TODO: ensure `config.json` specifies a number of training steps, learning # rate, and batch size in accordance with the project specifications. # You will not learn theory from this exercise, but you will glimpse # how, sometimes, programming is plumbing: connecting up the pipes # until the system works. try: for batch_index in range(get('TRAIN.NB_STEPS')): report_training_progress( batch_index, input_layer, loss_func, faces) batch = faces.train.next_batch(get('TRAIN.BATCH_SIZE')) optimizer.run(feed_dict={input_layer: batch[0]}) except KeyboardInterrupt: print('OK, I will stop training even though I am not finished.')
def compareDownsampling(): additionalArgs = [ { 'flag': '--scale', 'help': 'The scale at which to slice dimensions. For example, a scale of 2 means that each dimension will be devided into 2 distinct regions, for a total of 8 contiguous chunks.', 'action': 'store', 'type': int, 'dest': 'scale', 'required': True }, { 'flag': '--type', 'help': 'One of: depth, reverse, constant.', 'action': 'store', 'type': str, 'dest': 'type', 'required': True }, { 'flag': '--downscaleRate', 'help': 'One of 1, 2, 3.', 'action': 'store', 'type': int, 'dest': 'downscaleRate', 'required': True } ] ParseArgs('Run 3D CNN over structural MRI volumes', additionalArgs=additionalArgs) GlobalOpts.trainFiles = np.load(get('DATA.TRAIN_LIST')).tolist() GlobalOpts.valdFiles = np.load(get('DATA.VALD_LIST')).tolist() GlobalOpts.testFiles = np.load(get('DATA.TEST_LIST')).tolist() GlobalOpts.augment = 'none' GlobalOpts.name = '{}_scale{}_sample{}'.format(GlobalOpts.type, GlobalOpts.scale, GlobalOpts.downscaleRate) if GlobalOpts.downscaleRate == 1: GlobalOpts.imageBaseString = get('DATA.STRUCTURAL.NUMPY_PATH') GlobalOpts.imageBatchDims = (-1, 121, 145, 121, 1) elif GlobalOpts.downscaleRate == 2: GlobalOpts.imageBaseString = get('DATA.STRUCTURAL.DOWNSAMPLE_PATH') GlobalOpts.imageBatchDims = (-1, 61, 73, 61, 1) elif GlobalOpts.downscaleRate == 3: GlobalOpts.imageBaseString = get('DATA.STRUCTURAL.EXTRA_SMALL_PATH') GlobalOpts.imageBatchDims = (-1, 41, 49, 41, 1) GlobalOpts.trainBatchSize = 4 if GlobalOpts.type == 'depth': GlobalOpts.cnn = depthPatchCNN elif GlobalOpts.type == 'reverse': GlobalOpts.cnn = reverseDepthCNN elif GlobalOpts.type == 'constant': GlobalOpts.cnn = constantDepthCNN modelTrainer = ModelTrainer() GlobalOpts.summaryDir = '{}{}/'.format(get('TRAIN.CNN_BASELINE.SUMMARIES_DIR'), GlobalOpts.name) GlobalOpts.checkpointDir = '{}{}/'.format(get('TRAIN.CNN_BASELINE.CHECKPOINT_DIR'), GlobalOpts.name) RunTestOnDirs(modelTrainer)
def sign_in(self, onetime_code=None): """ Load token. If not available the user must provide a one time code from https://my.remarkable.com/connect/remarkable """ try: # Get device token if not stored local device_token = cfg.get("authentication.device_token") if device_token == None: if onetime_code is None or onetime_code == "": self.listener_handler.publish(EVENT_ONETIMECODE_NEEDED) return device_token = self._get_device_token(onetime_code) if device_token is None: self.listener_handler.publish(EVENT_DEVICE_TOKEN_FAILED) return # Renew the user token. user_token = self._get_user_token(device_token) if user_token is None: self.listener_handler.publish(EVENT_USER_TOKEN_FAILED) return # Save tokens to config auth = {"device_token": device_token, "user_token": user_token} cfg.save({"authentication": auth}) # Inform all subscriber self.listener_handler.publish(EVENT_SUCCESS, auth) except: auth = {} self.listener_handler.publish(EVENT_FAILED, auth) return auth
def SliceCNN(imagesPL, trainingPL, keepProbability=get('TRAIN.CNN_BASELINE.KEEP_PROB'), defaultActivation=tf.nn.elu, optionalHiddenLayerUnits=0, downscaleRate=None): with tf.variable_scope('ConvolutionalNetwork'): if imagesPL.dtype != tf.float32: imagesPL = tf.cast(imagesPL, tf.float32, name='CastInputToFloat32') if downscaleRate: if isinstance(downscaleRate, int): downscaleSize = [1, downscaleRate, downscaleRate, 1] imagesPL = pool2D(imagesPL, kernel_size=downscaleSize, strides=downscaleSize) elif isinstance(downscaleRate, (list, tuple)) and len(downscaleRate) == 2: downscaleSize = [1, downscaleRate[0], downscaleRate[1], 1] imagesPL = pool2D(imagesPL, kernel_size=downscaleSize, strides=downscaleSize) else: raise ValueError('Unrecognized downscale rate: {}'.format(downscaleRate)) ################## FIRST BLOCK ################## Block1 = block2D(imagesPL, trainingPL, blockNumber=1, filters=8) ################## SECOND BLOCK ################## Block2 = block2D(Block1, trainingPL, blockNumber=2, filters=16) ################## THIRD BLOCK ################## Block3 = block2D(Block2, trainingPL, blockNumber=3, filters=32) with tf.variable_scope('FullyConnectedLayers'): flattenedLayer = tf.layers.flatten(Block3) if optionalHiddenLayerUnits > 0: optionalHiddenLayer = standardDense(inputs=flattenedLayer, units=optionalHiddenLayerUnits, activation=defaultActivation, name='optionalHiddenLayer') droppedOutHiddenLayer = tf.contrib.layers.dropout(inputs=optionalHiddenLayer, keep_prob=keepProbability, is_training=trainingPL) flattenedLayer = droppedOutHiddenLayer numberOfUnitsInOutputLayer = 1 outputLayer = standardDense(flattenedLayer, units=numberOfUnitsInOutputLayer, activation=None, use_bias=False, name='outputLayer') return outputLayer
def get_url(section, image): # Get the url for the specific image command - from on config file url = None if section and image: url = config.get(section, image) + "?t=" + datetime.datetime.now().strftime("%d\%m\%Y-%H:%M:%S") return url
async def connect(self, hostname, port): try: hostname = hostname.encode('utf-8') except AttributeError: pass # config server = config.get('default', 'server') server_port = config.getint('default', 'server_port') loop = asyncio.get_event_loop() # 和proxyclient建立连接 try: transport, client = await loop.create_connection( ProxyClient, server, server_port) # 连接失败 except Exception: logging.error('Could not connect server: {}:{}'.format( server, server_port)) if self.transport.can_write_eof(): self.transport.write_eof() return False # 绑定server_transport和trasport client.server_transport = self.transport self.client_transport = transport # 发送地址信息, 域名和端口 self.client_transport.write( pack('!i%ssH' % len(hostname), len(hostname), hostname, port))
def sync(self): if self.state == model.item.STATE_SYNCING: return self.state = model.item.STATE_SYNCING self._update_state_listener() self._download_raw() self._write_remapy_file() self._update_state(inform_listener=False) annotations_exist = os.path.exists(self.path_rm_files) if self.type == TYPE_NOTEBOOK and annotations_exist: render.notebook(self.path, self.id(), self.path_annotated_pdf, self.is_landscape(), path_templates=cfg.get("general.templates")) else: if annotations_exist: # Also for epubs a pdf file exists which we can annotate :) # We will then show the pdf rather than the epub... render.pdf(self.path_rm_files, self.path_original_pdf, self.path_annotated_pdf, self.path_oap_pdf) self._update_state() self.parent().sync()
def test2(dataSet): print('----------------------------------------------------------------') print('----------------------------TEST 2------------------------------') print('----------------------------------------------------------------') X = np.array( [mat[np.tril_indices(mat.shape[0], k=-1)] for mat in dataSet.matrices]) Y = np.array(dataSet._df['AgeYears'].values.copy()) Y = np.reshape(Y, [Y.shape[0], 1]) imagesPL, predictionLayer = cnnNeural() labelsPL = tf.placeholder(tf.float32, shape=[None, 1]) lossFunction = tf.losses.mean_squared_error(labels=labelsPL, predictions=predictionLayer) global_step = tf.Variable(0, name='global_step', trainable=False) trainOperation = tf.train.AdamOptimizer( get('TRAIN.CNN.LEARNING_RATE')).minimize(lossFunction, global_step=global_step) RepeatModelXY(X, Y, imagesPL, labelsPL, predictionLayer, trainOperation, lossFunction, numRepeats=10)
def get_target_coverage(): global target_coverage if target_coverage is None: target_coverage = config.get() raise ValueError( "stats.get_target_coverage(): target_coverage not set") return target_coverage
def create_admin_user(): admin = User("administrator", "changeme", "The", "Administrator", config.get('admin', 'admin_email'), validated=True) session.add(admin) session.flush() admin_role = UserRoles(admin, "Administrator", False) session.add(admin_role) session.flush() for role in UserRoles.roles: if role != "Administrator": new_role = UserRoles(admin, role, True) session.add(new_role) session.flush() admin.add_change(admin) session.flush() session.commit() return admin
def _extract_features(self, gallery_images, config): # extract gallery features if config["dist_type"] == "hamming": gallery_features = np.zeros( [len(gallery_images), config['embedding_size'] // 8], dtype=np.uint8) else: gallery_features = np.zeros( [len(gallery_images), config['embedding_size']], dtype=np.float32) #construct batch imgs and do inference batch_size = config.get("batch_size", 32) batch_img = [] for i, image_file in enumerate(tqdm(gallery_images)): img = cv2.imread(image_file) if img is None: logger.error("img empty, please check {}".format(image_file)) exit() img = img[:, :, ::-1] batch_img.append(img) if (i + 1) % batch_size == 0: rec_feat = self.rec_predictor.predict(batch_img) gallery_features[i - batch_size + 1:i + 1, :] = rec_feat batch_img = [] if len(batch_img) > 0: rec_feat = self.rec_predictor.predict(batch_img) gallery_features[-len(batch_img):, :] = rec_feat batch_img = [] return gallery_features
def get_weights(saver, sess): ''' load model weights if they were saved previously ''' if is_file_prefix('TRAIN.CNN.CHECKPOINT'): saver.restore(sess, get('TRAIN.CNN.CHECKPOINT')) print('Yay! I restored weights from a saved model!') else: print('OK, I did not find a saved model, so I will start training from scratch!')
def setUpClass(cls): cls.logger = logging.getLogger(cls.__class__.__name__) cls.logger.setLevel(getattr(logging, config.get("log_level"))) stream_handler = logging.StreamHandler(sys.stdout) cls.logger.addHandler(stream_handler) cls.verify = verify.VerifyFuncs(cls.logger) cls.publisher = publisher.PublishFuncs(cls.logger)
def _change_ip(self): change_type = config.get('main', 'IP_CHANGE') if change_type == 'neo': zmiana() elif change_type == 'play': ipchange() else: raise UnrecognizedIpchangeType("Invalid IP_CHANGE value in configuration file.")
def on_orders_queue_declared(frame): """Called when a queue has been declared on the `orders` exchange.""" channel = config.get("channel") callbacks = dict(validated=process) queue_name = frame.method.queue assert queue_name in callbacks, "Unknown queue: %s" % queue_name channel.basic_consume(callbacks[queue_name], queue=queue_name) channel.queue_bind(exchange="orders", queue=queue_name)
def handle_ctl_msg(channel, method, header, body): """Called when we receive a control message from the shell.""" body = body.strip() print "* Rcvd ctrl msg: %s" % body channel.basic_ack(delivery_tag=method.delivery_tag) connection = config.get("connection") result_queue = config.get("result_queue") if body == "quit": channel.basic_publish(exchange="rpc", routing_key="jobs", body=body, properties=pika.BasicProperties(delivery_mode=1)) connection.close() connection.ioloop.start() else: channel.basic_publish(exchange="rpc", routing_key="jobs", body=body, properties=pika.BasicProperties( delivery_mode=1, reply_to=result_queue))
def on_incoming_queue_declared(frame): """Called when a queue has been declared on the `incoming` exchange.""" channel = config.get("channel") callbacks = dict(incoming=decrypt, decrypted=authenticate) queue_name = frame.method.queue assert queue_name in callbacks, "Unknown queue: %s" % queue_name channel.basic_consume(callbacks[queue_name], queue=queue_name) channel.queue_bind(exchange="incoming", queue=queue_name, routing_key="rk-%s" % queue_name)
def test_9_parallel_publication(self): dsets = datasets.get_parallel_test_datasets() pool_step = int(config.get('partest_pool_size_increment')) pool_max = int(config.get('partest_pool_size_max')) try: self.parallel_log_threshold = getattr(logging, config.get('partest_log_level')) except KeyError: pass pool_sizes = range(pool_step, 1 + pool_max, pool_step) self.ensure_empty(dset_list = dsets) try: self.run_parallel_tests(dsets, pool_sizes) finally: # the TearDownClass only removes the basic test data, not all the # datasets used in the parallel test, so do it here instead. self.ensure_empty(dset_list = dsets)
def on_result_queue_declared(frame): """ Called when the RPC results queue has been declared, the generated name is in the frame (response from RabbitMQ). """ result_queue = frame.method.queue config.set("result_queue", result_queue) channel = config.get("channel") channel.basic_consume(handle_result, queue=result_queue) channel.queue_bind(exchange="rpc", queue=result_queue, routing_key=result_queue)
def send_email(to, subject, html, sender=None): url = config.get('sendgrid', 'send_url') params = { 'api_user': env.get('SENDGRID_USER'), 'api_key': env.get('SENDGRID_KEY'), 'to': to, 'subject': subject, 'html': html, 'from': sender or '*****@*****.**' } rep = requests.get(url, params=params) rep.raise_for_status()
def pull_and_save_tweets(api): cfg = config.get("db") uri = "%s:%s@%s/%s" % ( cfg.get("user"), cfg.get("password"), cfg.get("host"), cfg.get("db")) conn = pymongo.Connection(uri) db = conn[cfg.get("db")] messages = db[cfg.get("messages_coll")] messages.ensure_index("id") maxids = db[cfg.get("maxids_coll")] cfg = config.get("twitter") since_id = get_since_id(maxids, cfg.get("user_name")) log.msg(pprint.pformat(since_id)) tweets = [] try: if since_id["id"]: tweets = api.getHomeTimeline(since_id=since_id["id"], count=199) else: tweets = api.getHomeTimeline(count=199) except Exception, e: log.err(e)
def send_email(to_addrs, subject, msg, from_addr, cc=None, bcc=None): """ Send an email """ msg['Subject'] = Header(subject, 'utf-8') msg['From'] = from_addr assert isinstance(to_addrs, list) msg['To'] = ', '.join(to_addrs) if cc: msg['CC'] = ', '.join(cc) to_addrs.extend(cc) if bcc: to_addrs.extend(bcc) smtp = smtplib.SMTP() smtp.connect(config.get('email', 'email_host')) smtp.sendmail(from_addr, to_addrs, msg.as_string()) smtp.quit()
async def connect(self, hostname, port): # config server = config.get('default', 'server') server_port = config.getint('default', 'server_port') loop = asyncio.get_event_loop() # 和proxyclient建立连接 try: transport, client = await loop.create_connection(ProxyClient, server, server_port) # 连接失败 except Exception: logging.error('Could not connect server: {}:{}'.format(server, server_port)) if self.transport.can_write_eof(): self.transport.write_eof() return False # 绑定server_transport和trasport client.server_transport = self.transport self.client_transport = transport # 发送地址信息, 域名和端口 self.client_transport.write( pack('!i%ssH' % len(hostname), len(hostname), hostname, port))
import calendar try: from utils import log, config except Exception, e: sys.path.append('../') from utils import log, config try: from mongo import mongo_commons except Exception, e: sys.path.append('../') from mongo import mongo_commons config = config.Config('crowddata') client = pymongo.MongoClient(config.get('database').get('connection')) database = config.get('database').get('db') document_data = config.get('database').get('document').get('data') document_date = config.get('database').get('document').get('date') def insertData(json): # insert (unique) date date = [int(s) for s in json['date'].split(',')] json["date"] = datetime.datetime(int(date[0]), int(date[1]), int(date[2])) date = [int(s) for s in json['fulldate'].split(',')] json["fulldate"] = datetime.datetime(int(date[0]), int(date[1]), int(date[2]), int(date[3]), int(date[4]), int(date[5])) # insert data json in data mongo_commons.insert(client, database, document_data, json) cursor = client[database][document_date].find({ "date" : json["date"]}).count()
def __init__(self): self._username = config.get('captcha', 'username') self._password = config.get('captcha', 'password')
def main(): server = GameServer(config.get('port'), GameClient) server.serve_forever()
collection = "layer" print sortdate result = db.find_by_code(collection, code, int(sortdate)) return Response(result, content_type='application/json; charset=utf-8') @app.route('/geometadata/query/<collection>', methods=['PUT']) @cross_origin(origins='*') def query_post(collection): # print collection # query = json.dumps(request.form['payload']) payload = json.loads(request.json['payload']) print payload result = db.find_query(collection, payload) print result return Response(result, content_type='application/json; charset=utf-8') @app.route('/geometadata/delete/<layername>', methods=['DELETE']) @cross_origin() def delete_layer( layername): return "TODO :Remove layername from layer and stats collections" if __name__ == '__main__': l.info(config.get('ip') + ':' + str(config.get('port'))) app.run(host=config.get('ip'), port=config.get('port'), debug=config.get('debug'))
if self.transport.can_write_eof(): self.transport.write_eof() return False # 绑定server_transport和trasport client.server_transport = self.transport self.client_transport = transport # 发送地址信息, 域名和端口 self.client_transport.write( pack('!i%ssH' % len(hostname), len(hostname), hostname, port)) if __name__ == '__main__': # config debug = config.getboolean('default', 'debug') local = config.get('default', 'local') local_port = config.getint('default', 'local_port') if debug: debug_level = logging.DEBUG else: debug_level = logging.ERROR # log logging.basicConfig(level=debug_level, format='%(threadName)10s %(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S', filemode='a+') logging.getLogger('asyncio').setLevel(debug_level) loop = asyncio.get_event_loop() if debug:
client.server_transport = self.transport self.client_transport = transport client.hostname = hostname # 返回给浏览器 hostip, port = transport.get_extra_info('sockname') host = unpack("!I", socket.inet_aton(hostip))[0] self.transport.write( pack('!BBBBIH', 0x05, 0x00, 0x00, 0x01, host, port)) if __name__ == '__main__': # config debug = config.getboolean('default', 'debug') server = config.get('default', 'server') server_port = config.getint('default', 'server_port') if debug: debug_level = logging.DEBUG else: debug_level = logging.ERROR # log logging.basicConfig(level=debug_level, format='%(threadName)10s %(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S', filemode='a+') logging.getLogger('asyncio').setLevel(debug_level) loop = asyncio.get_event_loop() if debug:
try: if since_id["id"]: tweets = api.getHomeTimeline(since_id=since_id["id"], count=199) else: tweets = api.getHomeTimeline(count=199) except Exception, e: log.err(e) log.msg(len(tweets)) if tweets: for tweet in tweets: if not isinstance(tweet, dict): continue try: tweet["text"] = urls.sanitize_urls(tweet["text"]) except: pass # not much we can do here messages.insert(tweets) ids = [int(tweet["id"]) for tweet in tweets if isinstance(tweet, dict)] if ids: since_id["id"] = max(ids) maxids.save(since_id) l = task.LoopingCall(pull_and_save_tweets, api) cfg = config.get("twitter") l.start(float(cfg.get("interval", 600))) # call every ten minutes # l.stop() will stop the looping calls reactor.run()
try: from utils import log, config except Exception, e: sys.path.append('../') from utils import log, config try: from mongo import mongo_commons except Exception, e: sys.path.append('../') from mongo import mongo_commons config = config.Config('geometadata') client = pymongo.MongoClient(config.get('database').get('connection')) database = config.get('database').get('db') document_layer = config.get('database').get('document').get('layer') document_stats = config.get('database').get('document').get('stats') """ Insert Layer Metadata in mongodb @param json: json data @return: id """ def insertMetadata(json): return mongo_commons.insert(client, database, document_layer, json) """ Delete Layer Metadata in mongodb @param json: json data
import requests, shlex, sys, json from utils import message_printer, get_location, gen_map, prettyprint_map, reports_printer, comments_printer from utils import config APIBASE = "http://s.jdiez.me:5000/json/" APIBASE = config.get("api", "base") print "euskalMap CLI consumer - v.0.1" print def help(arg): print "Available commands: " print print ', '.join(commands.keys()) # weird python syntax. lol def request_error(code, response): print "! API returned code " + str(code) + " and data:" print response def show(arg): def messages(arg): def print_data(url): data = requests.get(url) if data.status_code != 200: request_error(data.status_code, data.text) data = json.loads(data.text) message_printer(data) additional_arguments = { 'timestamp': 'has_timestamp',
# we make sure that rating is a num between 0-100 try: float(rating) except TypeError: rating = None if not comment: raise ValidationError('Must provide comment to add') # get the image id # we are going to use the json map to # retreive the path to the image media_id = form.getvalue('mid') # lookup the media path lookup_path = config.get('map_path') resource_map = json.load(open(lookup_path,'r')) path = resource_map.get(media_id) if not path: raise ValidationError('Media not found') set_image_comment(path,body=comment,rating=rating) # where do we point them back to? that medias page page_url = get_media_page_url(media_id) # lets update the pages associated w/ that id page_writer = PageWriter(template_root=config.get('template_root'), pages_root=config.get('pages_root'), media_dir_template=config.get('media_dir_template'))
def on_job_queue_declared(frame): """Called when the RPC job queue has been declared.""" channel = config.get("channel") channel.basic_consume(handle_job, queue="jobs") channel.queue_bind(exchange="rpc", queue="jobs", routing_key="jobs")
import csv try: from utils import log, config except Exception, e: sys.path.append('../../') from utils import log, config try: from postgresql.crowddata.crowddata import DBCrowddata except Exception, e: sys.path.append('../../') from postgresql.crowddata.crowddata import DBCrowddata config = config.Config('crowddata_postgresql') database = config.get('database') print database db = DBCrowddata(database) l = log.Logger() class RandomData: def __init__(self): return None def date(self, start, end): delta = end - start int_delta = (delta.days * 24 * 60 * 60) + delta.seconds random_second = randrange(int_delta) return start + timedelta(seconds=random_second)
def on_ctl_queue_declared(_): """Called when the control queue has been declared.""" channel = config.get("channel") channel.basic_consume(handle_ctl_msg, queue='rpc_ctl')
import pymongo try: from utils import log, config except Exception, e: sys.path.append('../') from utils import log, config try: from mongo import mongo_commons except Exception, e: sys.path.append('../') from mongo import mongo_commons config = config.Config('geostatistics') client = pymongo.MongoClient(config.get('geostats_datastore').get('connection')) database = config.get('geostats_datastore').get('db') """ Delete Layer Metadata in mongodb @param json: json data @return: id """ def removeMetadata(layer, json): return mongo_commons.remove(client, database, layer, json) """ Insert Layer Statistics in tmongodb @param json: json data @return: id
#!/usr/bin/env python from utils import config from utils.esg_config import default_config_path as esgini from one_time_setup.simple_mapfile_gen import gen_all_mapfiles from one_time_setup.make_dummy_ensemble import MakeDummyEnsemble mapfile_dir = config.get('test_mapfile_dir') data_root = config.get('test_data_dir') host_certs_dir = config.get('host_certs_dir') print "making ensemble for parallel test" ensmaker = MakeDummyEnsemble(int(config.get('partest_ensemble_size')), config.get('partest_ensemble_dir'), config.get('partest_template_member'), config.get('partest_member_pattern')) ensmaker.make_ensemble() print "making mapfiles" gen_all_mapfiles(data_root, mapfile_dir) print """ Mapfiles for test publication are set up. You also need to: (1) in %s: (a) ensure that the following directory is listed in under thredds_dataset_roots: