def get_api_data(self): """ Sends API request to IGDB. If response is equal to limit of 50, sends another until complete list is retrieved or limit of 200 is reached. """ try: response = requests.request("GET", self.url, headers=self.headers, data=self.payload) response_list = yaml.load(response.text) while len(yaml.load(response.text)) is self.limit and ( self.offset + self.limit) <= 150: self.offset += self.limit self.payload = "{} limit {}; offset {};".format( self.search, self.limit, self.offset) response = requests.request("GET", self.url, headers=self.headers, data=self.payload) response_list += yaml.load(response.text) except Exception as e: logging.Exception("Exception with payload: " + self.payload) logging.Exception(str(e)) raise return response_list
def init_conn(db): con = None try: con = sqlite3.connect(db) except sqlite3.Error as e: logging.Exception("Database error: %s" % e) except Exception as e: logging.Exception("Exception in _query: %s" % e) return con
def connect(self): try: print self.client[str(self.db_name)] except Exception as ex: print logging.Exception(ex) return False return True
def mc_clean(): try: mc = memcache.Client(["localhost:11211"], debug=1) return mc.flush_all() except Exception as m: log.Exception(m) raise SystemExit, 1
def on_stop_recording(self, task): """Notification that we are done with recording""" import psutil if self.cpu_start is not None: cpu_end = psutil.cpu_times() cpu_busy = (cpu_end.user - self.cpu_start.user) + \ (cpu_end.system - self.cpu_start.system) cpu_total = cpu_busy + (cpu_end.idle - self.cpu_start.idle) cpu_pct = cpu_busy * 100.0 / cpu_total task['page_data']['fullyLoadedCPUms'] = int(cpu_busy * 1000.0) task['page_data']['fullyLoadedCPUpct'] = cpu_pct self.cpu_start = None self.recording = False if self.thread is not None: self.thread.join(10) self.thread = None # record the CPU/Bandwidth/memory info if self.usage_queue is not None and not self.usage_queue.empty( ) and task is not None: file_path = os.path.join(task['dir'], task['prefix']) + '_progress.csv.gz' gzfile = gzip.open(file_path, GZIP_TEXT, 7) if gzfile: gzfile.write( "Offset Time (ms),Bandwidth In (bps),CPU Utilization (%),Memory\n" ) try: while True: snapshot = self.usage_queue.get(5) if snapshot is None: break gzfile.write('{0:d},{1:d},{2:0.2f},-1\n'.format( snapshot['time'], snapshot['bw'], snapshot['cpu'])) except Exception: logging.Exception("Error processing usage queue") gzfile.close() if self.tcpdump is not None: logging.debug('Waiting for tcpdump to stop') from .os_util import wait_for_all if platform.system() == 'Windows': wait_for_all('WinDump') else: wait_for_all('tcpdump') self.tcpdump = None if self.ffmpeg is not None: logging.debug('Waiting for video capture to finish') if platform.system() == 'Windows': self.ffmpeg.communicate(input='q'.encode('utf-8')) else: self.ffmpeg.communicate(input='q') self.ffmpeg = None if platform.system() == 'Windows': from .os_util import kill_all kill_all('ffmpeg.exe', True) else: subprocess.call(['killall', '-9', 'ffmpeg']) self.job['shaper'].reset()
def _delete_local(local_base, files, dryrun): for name, mtime, size in files: logging.info("rem_: %s/%s" % (local_base, name)) if dryrun: continue local_path = os.path.join(local_base, name) try: os.remove(local_path) except Exception as e: logging.Exception("unable to delete: %s" % local_path)
def insert_tag(self, tag_name, is_company=0): try: c = self.conn.cursor() c.execute('''INSERT OR IGNORE INTO cup_tag VALUES(?,?)''', (tag_name, is_company)) # self.conn.commit() except: logging.Exception('Failed to insert tag: ' + str(sys.exc_info())) raise finally: c.close()
def insert_question_with_tag(self, question_id, tag_name): try: c = self.conn.cursor() c.execute( '''INSERT OR IGNORE INTO cup_question_with_tag VALUES(?, ?)''', (question_id, tag_name)) # self.conn.commit() except: logging.Exception('Failed to insert question_with_tag: ' + str(sys.exc_info())) raise finally: c.close()
def extract_rois((full_path_to_file, individualframename)): # Create SIMA Sequence & ImagingDataset objects from image file(s) or # motion correction if action == 'extract', assume that motion # correction was done on EC2 previously if action == 'both': try: motion_correction((full_path_to_file, individualframename)) except Exception as e: print('Motion correction failed') print e logging.Exception('Motion correction failed') filename = os.path.splitext(os.path.basename(full_path_to_file))[0] dataset = sima.ImagingDataset.load(filename + '_mc.sima') # Obtain ROIs if to_segment: logging.info("Segmenting images for %s..." % filename) # Automated segmentation # Define segmentation method and post processing. segment_approach = sima.segment.PlaneNormalizedCuts() segment_approach.append(sima.segment.SparseROIsFromMasks()) segment_approach.append(sima.segment.SmoothROIBoundaries()) segment_approach.append(sima.segment.MergeOverlapping(threshold=0.5)) # Apply segmentation to dataset rois = dataset.segment(segment_approach) logging.info("Done segmenting images for %s" % filename) print("Done segmenting images for %s" % filename) else: logging.info("Importing ROIs from ImageJ for %s..." % filename) print("Importing ROIs from ImageJ for %s..." % filename) # Load ROIs from ImageJ rois = ROIList.load(filename + '_mc_' + roi_filename, fmt='ImageJ') dataset.add_ROIs(rois, 'from_ImageJ') logging.info("Done importing ROIs for %s" % filename) print("Done importing ROIs for %s" % filename) # Extract signals from ROIs into numpy file signals = dataset.extract(rois) extracted_signals = np.asarray(signals['raw']) np.save(filename + '_extractedsignals', extracted_signals) logging.info("Done extracting signals") print("Done extracting signals")
def test(): clf3 = joblib.load("save/clf.pkl") test_images = get_images("test_images/") for each in test_images: img = skimage.io.imread(each) img = get_image_feature(img) img_resized = resize(img, dimension, anti_aliasing=True, mode='reflect') flat_data.append(img_resized.flatten()) try: print(clf3.predict(img_resized.flatten())) except Exception as e: import logging logging.Exception(e) flat_data = np.array(flat_data) print(clf3.predict(flat_data[0:5]))
def insert_question(self, question): last_id = -1 try: c = self.conn.cursor() c.execute( '''INSERT OR IGNORE INTO cup_question VALUES(NULL, ?, ?, ?, ?, ?, ?, datetime('now'), 0 )''', (question.question_id, question.question_content, question.link, question.up_votes, question.comment_count, question.creation_date)) # self.conn.commit() last_id = c.lastrowid except: logging.Exception('Failed to insert question: ' + str(sys.exc_info())) raise finally: c.close() return last_id
def parse_pe_pkl(file_index, file_id, fjson, unprocessed): """ Function to parse pickle file to find the boundaries of PE sections in a sample's pickle representation Args: file_index: PE sample index file_id: PE name fjson: pickle data representation of PE sample unprocessed: keeps track of count of samples not processed properly Returns: section_bounds: PE section boundaries unprocessed: keeps track of count of samples not processed properly file_byte_size: size of full sample """ section_bounds = [] file_byte_size = None max_section_end_offset = 0 try: # file_byte_size = fjson['size_byte'] with open(cnst.RAW_SAMPLE_DIR + file_id, 'rb') as f: file_byte_size = len(list(f.read())) pe = pefile.PE(cnst.RAW_SAMPLE_DIR + file_id) for pkl_section in pe.sections: section_bounds.append( (pkl_section.Name.strip(b'\x00').decode("utf-8").strip(), pkl_section.PointerToRawData, pkl_section.PointerToRawData + pkl_section.SizeOfRawData)) if (pkl_section.PointerToRawData + pkl_section.SizeOfRawData) > max_section_end_offset: max_section_end_offset = (pkl_section.PointerToRawData + pkl_section.SizeOfRawData) # Placeholder section "padding" - for activations in padding region # if max_section_end_offset < fjson["size_byte"]: # section_bounds.append((cnst.TAIL, max_section_end_offset + 1, fjson["size_byte"])) # section_bounds.append((cnst.PADDING, fjson["size_byte"] + 1, cnst.MAX_FILE_SIZE_LIMIT)) except Exception as e: logging.Exception("parse failed . . . [FILE INDEX - " + str(file_index) + "] [" + str(file_id) + "] ") unprocessed += 1 return section_bounds, unprocessed, file_byte_size
def on_stop_recording(self, task): """Notification that we are done with an operation that needs to be recorded""" if self.thread is not None: self.thread.join(10) self.thread = None # record the CPU/Bandwidth/memory info self.recording = False if self.usage_queue is not None and not self.usage_queue.empty( ) and task is not None: file_path = os.path.join(task['dir'], task['prefix']) + '_progress.csv.gz' gzfile = gzip.open(file_path, GZIP_TEXT, 7) if gzfile: logline = "Offset Time (ms),Bandwidth In (bps),CPU Utilization (%),Memory\n" logging.debug(logline) gzfile.write(logline) try: while True: snapshot = self.usage_queue.get(5) if snapshot is None: break logline = '{0:d},{1:d},-1,-1\n'.format( snapshot['time'], snapshot['bw']) logging.debug(logline) gzfile.write(logline) except Exception: logging.Exception("Error processing usage queue") gzfile.close() if self.tcpdump_enabled: tcpdump = os.path.join(task['dir'], task['prefix']) + '.cap' self.adb.stop_tcpdump(tcpdump) if self.video_enabled and not self.job['disable_video']: task['video_file'] = os.path.join(task['dir'], task['prefix']) + '_video.mp4' self.adb.stop_screenrecord(task['video_file']) self.job['shaper'].reset()
def create_database(self): try: c = self.conn.cursor() c.execute('''CREATE TABLE IF NOT EXISTS cup_question ( question_id INTEGER PRIMARY KEY ON CONFLICT IGNORE, question_identifier TEXT UNIQUE NOT NULL, content TEXT NOT NULL, link TEXT, up_votes INTEGER, comment_count INTEGER, creation_date DATE, creation_time_in_db DATE, is_deleted INTEGER )''') c.execute('''CREATE TABLE IF NOT EXISTS cup_tag ( name TEXT PRIMARY KEY, is_company INTEGER )''') c.execute('''CREATE TABLE IF NOT EXISTS cup_question_with_tag ( question_id INTEGER, tag_name TEXT, PRIMARY KEY (question_id, tag_name) )''') self.conn.commit() logging.info('Database initialized.') except: logging.Exception('Failed to create databse: ' + str(sys.exc_info())) raise finally: c.close()
p_icd = df_grouped_icd.loc[df_grouped_icd['dsysrtky'] == p_info['dsysrtky']] df_p = prepare_data(p_info, p_icd.iloc[0], window_back, window_forward, shift) frame.append(df_p) df_all = pd.concat(frame, ignore_index=True) return df_all try: df_training = combine_patient(tr, df_grouped_icd, window_back, window_forward, shift) df_testing = combine_patient(ts, df_grouped_icd, window_back, window_forward, shift_test) except Exception as e: logging.Exception("Unexpected exception! %s", e) print(df_training.shape) print(df_testing.shape) save_csv = 1 if save_csv == 1: training_file = dir_csv + 'blance' + str( balance) + testing_delay_indicate + 'new_df_training' + str( window_forward) + '_' + str(shift) + '.csv' testing_file = dir_csv + 'blance' + str( balance) + testing_delay_indicate + 'new_df_training' + str( window_forward) + '_' + str(shift) + '.csv' df_training.to_csv(training_file) df_testing.to_csv(testing_file) logging.critical('training data is saved to file' + training_file)