def __init__(self, entity_name=None, source_language_script=ENGLISH_LANG, translation_enabled=False): """ Initializes a TextDetector object with given entity_name Args: entity_name: A string by which the detected substrings that correspond to text entities would be replaced with on calling detect_entity() source_language_script: ISO 639 code for language of entities to be detected by the instance of this class translation_enabled: True if messages needs to be translated in case detector does not support a particular language, else False """ # assigning values to superclass attributes self._supported_languages = [ENGLISH_LANG, HINDI_LANG] super(TextDetector, self).__init__(source_language_script, translation_enabled) self.text = None self.text_dict = {} self.tagged_text = None self.text_entity_values = [] self.original_texts = [] self.processed_text = None self.entity_name = entity_name self.tag = '__' + self.entity_name + '__' # defaults for auto mode self._fuzziness = "auto:4,7" self._fuzziness_lo, self._fuzziness_hi = 4, 7 self._min_token_size_for_fuzziness = self._fuzziness_lo # self.set_fuzziness_threshold(fuzziness=(self._fuzziness_lo, self._fuzziness_hi)) # defaults for non-auto mode self.set_fuzziness_threshold(fuzziness=1) self._min_token_size_for_fuzziness = 4 self.db = DataStore()
def test_get_references(self): ds = DataStore('testing/data/reference_types') with self.assertRaises(NotAManagedType): ds.type('XXX') references = ds.type('Type1') self.assertTrue(hasattr(references, 'get'), 'References does not include a get attribute') self.assertTrue(callable(getattr(references, 'get')), 'References.get is not callable') self.assertTrue(hasattr(references, 'add'), 'References does not include a add attribute') self.assertTrue(callable(getattr(references, 'add')), 'References.add is not callable') self.assertTrue(hasattr(references, 'update'), 'References does not include a update attribute') self.assertTrue(callable(getattr(references, 'update')), 'References.update is not callable') self.assertTrue(hasattr(references, 'filter'), 'References does not include a filter attribute') self.assertTrue(callable(getattr(references, 'filter')), 'References.filter is not callable') self.assertTrue(hasattr(references, 'delete'), 'References does not include a delete attribute') self.assertTrue(callable(getattr(references, 'delete')), 'References.delete is not callable')
class EncryptorWatcher(LoggingEventHandler): ''' This class enters all file 'created' events to a database pointed to by dbFolder ''' def __init__(self, pathStructure, dbFolder): super(LoggingEventHandler, self).__init__() self.pathStructure = pathStructure self.dataStore = DataStore(dbFolder) def on_modified(self, event): path = os.path.join(self.pathStructure['inBox'], event.src_path) logging.debug("encryptorWatch on_modified file") info = "Modified: " + event.src_path + " " + str(os.path.getsize(path)) logging.debug(info) def on_created(self, event): path = os.path.join(self.pathStructure['inBox'], event.src_path) if os.path.isdir(os.path.abspath(event.src_path)): logging.debug('WatchProcess: Folder Encryption is not supported.') return self.dataStore.addFilePathToDataBaseStoreWithType(os.path.abspath(event.src_path), self.pathStructure['watchType'], self.pathStructure['name']) info = "Created: " + event.src_path + " " + str(os.path.getsize(path)) logging.debug("encryptorWatch on_created file") logging.debug(info)
def test_filter_reference(self): ds = DataStore('testing/data/reference_types') results = ds.type('Type1').filter(name='Item_1') self.assertEqual(1, len(results)) self.assertEqual('1', results[0].ref) self.assertEqual('Item_1', results[0].name) self.assertEqual('Item 1', results[0].description)
def test_save_empty(self): table = self.create_session_table() d = DataStore('dynamodb', 'Sessions', 'PatientId', 'SessionId') response = d.save(SESSION_INFO_EMPTY) self.assertEqual('BAD', response['status']) response = table.scan() self.assertEqual(0, len(response['Items']))
def handle(config): try: flagcheck=True logger.info("calling jmx for metrics") #check for jmx hosts file #TODO add code for handling metrics from multiple JMX hosts # #JAVA(libjvm='./lib/jmx/libjvm.so') #JAVA() jpype.attachThreadToJVM() jmx=JMX(host='96.119.153.107',port=9999) DS=DataStore() for condition in config.get('if').get('jmx'): baseline=DS.getbaseline(condition) current=jmx.get_attr(condition.get('object'),condition.get('type'),condition.get('attribute')) logger.debug(current) logger.debug(str(current) + condition.get('operator') + repr(baseline)) out=eval(str(current) + condition.get('operator') + repr(baseline)) if not bool(out): flagcheck=False break DS.setbaseline(current.floatValue(),baseline,condition) del jmx return flagcheck except Exception,e: print "in exception" print e logger.error(e) return False
class App(tk.Frame): def __init__(self, parent, *args, **kwargs): tk.Frame.__init__(self, parent, *args, **kwargs) self.parent = parent self.datastore = DataStore() self.plclink = PLCLink() self.varlist = getvars() # Two frames, display frame and control frame displayfrm = PlotWidget(self, relief=RAISED, borderwidth=1) displayfrm.pack(fill=BOTH, expand=1) ctrlfrm = ControlWidget(self) ctrlfrm.pack(fill=BOTH, expand=1) def logger(self): try: data = self.plclink.getdata() self.datastore.insert(data) except: print("Logging failed on {}".format(strftime("%b %d %Y %H:%M:%S", localtime()))) finally: # Log again after specified interval self.master.after(loginterval, self.logger) def setplot(self): PlotConfigurator(self, self.varlist)
def __init__(self): self.state = State.Init self.config = { 'hover_height': 1.0, # States for which movement.fix_hover() will NOT be called (to make sure the drone is at `hover_height`) 'exclude_from_fix_hover': [ State.Init, State.Takeoff, State.Land, State.Done, ], # Radius in meters around a blacklisted goal that the robot will ignore 'blacklisted_goal_radius': 2.0 } self.store = DataStore() self.movement = MovementHandler(core=self, datastore=self.store) self.actions = Actions(core=self, datastore=self.store, movement_handler=self.movement) self.planner = Planner(core=self, datastore=self.store, movement_handler=self.movement) # Aux files self.temp_data = {} self.last_goal = None
class singleFileWatcher(LoggingEventHandler): ''' This class enters all file 'created' events to a database pointed to by dbFolder ''' def __init__(self, pathStructure, dbFolder): super(LoggingEventHandler, self).__init__() self.pathStructure = pathStructure self.dataStore = DataStore(dbFolder) def on_created(self, event): for ignoreFile in ['.DS_Store', 'Thumbs.db']: if ignoreFile in os.path.abspath(event.src_path): info = 'File ignored: ' + os.path.abspath(event.src_path) logging.debug(info) return info = 'On created: ' + os.path.abspath(event.src_path) logging.debug(info) if os.path.isdir(os.path.abspath(event.src_path)): info = 'Directory analysis is not available' logging.debug(info) return self.dataStore.addFileToDatabase(os.path.abspath(event.src_path)) info = 'adding ' + event.src_path + ' to the database' logging.debug(info)
def close(self): self.commitStockBatch() self.commitBatch() self.conn.commit() self.conn.close() self.conn = None DataStore.close(self)
def getcompany_info(self, name, url): logger = Logger(logname='error.log', logger="58com").getlog() ds = DataStore() try: company_text = [] html = proxy.proxy_request(url) soup = BeautifulSoup(html, 'html.parser') tag = soup.find(class_="basicMsg") ul = tag.find("ul") li_tags = ul.find_all(name='li') strinfo = re.compile('\s') for li in li_tags: txt = strinfo.sub('', li.get_text()) company_text.append(txt.split(':')[1]) #获取工商信息 #gongshang_info = tianyan.tianyan_search(name) #gongshang_info = ','.join(gongshang_info) ds.insert_database(name, company_text) except urllib.error.URLError as e: if hasattr(e, "code"): print(e.code) if hasattr(e, "reason"): print(e.reason) logger.error("Get company info fail, company name: %s, url: %s", name, url) #记录解析失败的公司和url except Exception as e: print("exception:" + str(e)) sleep(1)
def handle(self, *args, **options): entity_data_directory_path = None csv_file_paths = None if ('entity_data_directory_path' in options and options['entity_data_directory_path']) or \ ('csv_file_paths' in options and options['csv_file_paths']): if 'entity_data_directory_path' in options and options['entity_data_directory_path']: entity_data_directory_path = options['entity_data_directory_path'] if not os.path.exists(entity_data_directory_path): entity_data_directory_path = None if 'csv_file_paths' in options and options['csv_file_paths']: csv_file_paths = options['csv_file_paths'].split(',') csv_file_paths = [csv_file_path for csv_file_path in csv_file_paths if csv_file_path and csv_file_path.endswith('.csv')] db = DataStore() db.repopulate(entity_data_directory_path=entity_data_directory_path, csv_file_paths=csv_file_paths) if entity_data_directory_path: self.stdout.write( 'Successfully Repopulated entity data from csv files at "%s"' % entity_data_directory_path) if csv_file_paths: self.stdout.write('Successfully Repopulated entity data from %d other csv file(s)"' % len(csv_file_paths)) else: self.stdout.write(self.style.ERROR('argument --entity_data_directory_path or --csv_file_paths required'))
def test_add_reference_raises_DuplicateReference(self): ds = DataStore('testing/data/reference_types') type1 = ds.type('Type1').get("1") with self.assertRaises(Type1.DuplicateReference): x = ds.type('Type1').add(type1) with self.assertRaises(Type1.DuplicateReference): x = ds.type('Type1').add({}, ref="1")
def test_delete_reference_raises_DoesNotExist(self): ds = DataStore('testing/data/reference_types') typex = Type1(ref='X', name='Type_X', desc='Type X') with self.assertRaises(Type1.DoesNotExist): x = ds.type('Type1').delete(typex) with self.assertRaises(Type1.DoesNotExist): x = ds.type('Type1').delete({}, ref="X")
def __init__(self, entity_name=None): """ Initializes a TextDetector object with given entity_name Args: entity_name: A string by which the detected substrings that correspond to text entities would be replaced with on calling detect_entity() """ self.text = None self.regx_to_process = Regex([(r'[\'\/]', r'')]) self.text_dict = {} self.tagged_text = None self.text_entity = [] self.original_text_entity = [] self.processed_text = None self.entity_name = entity_name self.tag = '__' + self.entity_name + '__' # defaults for auto mode self._fuzziness = "auto:4,7" self._fuzziness_lo, self._fuzziness_hi = 4, 7 self._min_token_size_for_fuzziness = self._fuzziness_lo # self.set_fuzziness_threshold(fuzziness=(self._fuzziness_lo, self._fuzziness_hi)) # defaults for non-auto mode self.set_fuzziness_threshold(fuzziness=1) self._min_token_size_for_fuzziness = 4 self.db = DataStore()
def main(): """Parse the command line arguments, expecting one of the following formats: -) (-i ChannelID | -u Username) (add | check | remove) -) check | list and perform the appropriate action """ parser = get_parser() args = parser.parse_args() youtube = YouTube() store = DataStore('%s-data.sqlite3' % sys.argv[0], 'schema.sql') channel = None if args.username is not None: channel = youtube.get_channel_by_username(args.username) elif args.id is not None: channel = youtube.get_channel_by_id(args.id) if args.action == 'add': store.store_channel(channel) elif args.action == 'remove': store.remove_channel(channel) elif args.action == 'list': data = [] for item in store.get_channels(): data.append([ item['id'], item['title'], arrow.get(item['added_on']).humanize(), arrow.get(item['last_checked']).humanize() ]) pretty_print(['ID', 'Title', 'Added', 'Last Checked'], data) elif args.action == 'check': # If the user passed a specific channel, check for new uploads # otherwhise check for uploads from every previously added channel channels = [] if channel is not None: channels.append(store.get_channel_by_id(channel['id'])) else: channels = store.get_channels() data = [] to_check = dict() for channel_item in channels: to_check[channel_item['id']] = channel_item['last_checked'] uploads = youtube.get_uploads(to_check) for upload in uploads: data.append([ upload['channel_title'], upload['title'], arrow.get(upload['published_at']).humanize(), 'https://youtube.com/watch?v=%s' % (upload['id'], ) ]) pretty_print(['Channel', 'Title', 'Published', 'Link'], data) for channel_id in to_check.keys(): store.update_last_checked(channel_id)
class Achievements: __borg_state = {} __achievements_id = "account_achievements" def __init__(self, config): self.__dict__ = self.__borg_state self.config = config self.api = GW2_API(config) self.ds = DataStore(config) self.render = Render(config) def _get_current(self): """Get the current achievements for the character.""" cheeves = self.api.get("account/achievements") cheeves_by_id = {} for cheeve in cheeves: cheeves_by_id[cheeve.id] = cheeve return cheeves_by_id def _get_new_unlock_cheeves(self, old_cheeves, new_cheeves): """Given a dict of old and new Achievements, find those that are newly unlocked. Returns a tuple of: (unlocks, newness) where -unlocks is newly completed cheeves -newness is new added cheeves """ unlocks = [] newness = [] for cheeve in new_cheeves: if cheeve['id'] not in old_cheeves: newness.append(cheeve) elif cheeve['done'] != old_cheeves[cheeve['id']]['done']: unlocks.append(cheeve) return (unlocks, newness) def _get_new_progress_cheeves(self, old_cheeves, new_cheeves): """Given a dict of old and new Achievements, find those that have new progress on them.""" new_prog = [] for cheeve in new_cheeves: if cheeve.get('current', 0) != \ old_cheeves[cheeve['id']].get('current', 0): new_prog.append(cheeve) def update(self, cheeves=None): """Will update the datastore with the current cheeves. Intended to be called once per day, per week, per cycle (whatever). If 'cheeves' is ommitted, will get the current cheevese via API""" if cheeves is None: cheeves = self._get_current() self.ds.put(self.__achievements_id, cheeves)
def __init__(self, prodj): super().__init__() self.prodj = prodj self.queue = Queue() self.pdb_enabled = True self.pdb = PDBProvider(prodj) self.dbc_enabled = True self.dbc = DBClient(prodj) # db queries seem to work if we submit player number 0 everywhere (NOTE: this seems to work only if less than 4 players are on the network) # however, this messes up rendering on the players sometimes (i.e. when querying metadata and player has browser opened) # alternatively, we can use a player number from 1 to 4 without rendering issues, but then only max. 3 real players can be used self.own_player_number = 0 self.request_retry_count = 3 self.metadata_store = DataStore( ) # map of player_number,slot,track_id: metadata self.artwork_store = DataStore( ) # map of player_number,slot,artwork_id: artwork_data self.waveform_store = DataStore( ) # map of player_number,slot,track_id: waveform_data self.preview_waveform_store = DataStore( ) # map of player_number,slot,track_id: preview_waveform_data self.beatgrid_store = DataStore( ) # map of player_number,slot,track_id: beatgrid_data
def perform_search(search_str,results_collector): base_url = 'http://sfbay.craigslist.org' cl_html_results = requests.get(search_str) soup = BeautifulSoup(cl_html_results.text, 'html.parser') ds = DataStore(storetype='sql') for result in soup.find_all(attrs={"data-pid": re.compile('\d+')}): link_title = result.find(id='titletextonly') if link_title is None: # print "Cannot find title for entry %s", result next else: datapid = result.attrs['data-pid'] link_title_text = link_title.text link = '{0}{1}'.format(base_url, result.find('a').attrs['href']) #print "debug: {0} | {1} | {2}".format(datapid, link_title_text, link) ds.save_entry(datapid=datapid, title=link_title_text, url=link) for i in ds.new_listings: results_collector.append(i)
def handle(self, *args, **options): db = DataStore() if db.exists(): db.delete() self.stdout.write('Successfully deleted DataStore structure and data.') else: self.stdout.write('DataStore structure not created. Skipping delete action')
def handle(self, *args, **options): if 'entity_name' in options and options['entity_name']: entity_name = options['entity_name'] db = DataStore() db.delete_entity(entity_name=entity_name) self.stdout.write('Successfully deleted entity data for "%s"' % entity_name) else: self.stdout.write(self.style.ERROR('argument --entity_name required'))
def test1(self): con = make_dbcon() ds = DataStore(con) col = ds.collection("users") i = Index(con, col, 'email') self.assertEqual(i.name(), 'email')
def unknown_negative(): global ds _ds = DataStore('/tmp/pood/ds') paths = _ds.fetch(1).all().paths() if len(paths) > 0: path = paths[0] ds.store(0).tile(Image.open(open(path, mode='rb')), 100, size=(16, 16)) os.unlink(path) return redirect('/')
def handle_get(key): """Return a tuple containing True if the key exists and the message to send back to the client.""" if key not in POROCESSING: ds = DataStore() data = ds.get(key) if data: return(True, (data[0],data[1])) return(False, 'ERROR: Key [{}] not found'.format(key))
def update_poi(id): store = DataStore() poi = PointOfInterest.from_request(request) poi.id = id valid = poi.validate() if len(valid) == 0: poi = store.update_poi(poi) return to_json(poi) else: return to_json(valid)
def handle(self, *args, **options): db = DataStore() if db.exists(): self.stdout.write( 'DataStore structure already created, skipping this action. ' 'If you want to recreate it, run python manage.py delete_datastore first.' ) else: db.create() self.stdout.write('Successfully created DataStore structure')
def test_save(self): table = self.create_session_table() d = DataStore('dynamodb', 'Sessions', 'PatientId', 'SessionId') response = d.save(SESSION_INFO_1) self.assertEqual('OK', response['status']) response = table.scan() item = response['Items'][0] self.assertTrue( 'ComprehendS3Path' in item[DATASTORE_COLUMN_COMPREHEND_S3_PATH]) self.assertEqual('p-1', item[DATASTORE_COLUMN_PATIENT_ID])
def api_home(): request_info = extract_request(request, session) if request.method == 'POST': message = request.form.get('message', '') datastore = request.form.get('datastore', '') DataStore.store_message(datastore, message) context = {'request_info': request_info} response = make_response(json.dumps(context)) response.set_cookie('dt', get_now()) return response
def __init__(self, entity_name=None, source_language_script=lang_constant.ENGLISH_LANG, translation_enabled=False): """ Initializes a TextDetector object with given entity_name Args: entity_name: A string by which the detected substrings that correspond to text entities would be replaced with on calling detect_entity() source_language_script: ISO 639 code for language of entities to be detected by the instance of this class translation_enabled: True if messages needs to be translated in case detector does not support a particular language, else False """ # assigning values to superclass attributes self._supported_languages = [ lang_constant.ENGLISH_LANG, lang_constant.HINDI_LANG, lang_constant.GUJARATI_LANG, lang_constant.HINDI_LANG, lang_constant.ENGLISH_LANG, lang_constant.GUJARATI_LANG, lang_constant.BENGALI_LANG, lang_constant.MARATHI_LANG, lang_constant.TELEGU_LANG, lang_constant.TAMIL_LANG, lang_constant.URDU_LANG, lang_constant.KANNADA_LANG, lang_constant.ORIYA_LANG, lang_constant.MALAYALAM_LANG, lang_constant. PUNJABI_LANG # Added temporarily till text detection is ported to v2 api ] super(TextDetector, self).__init__(source_language_script, translation_enabled) self.tagged_text = None self.processed_text = None self.__texts = [] self.__tagged_texts = [] self.__processed_texts = [] self.entity_name = entity_name self.tag = '__' + self.entity_name + '__' # defaults for auto mode self._fuzziness = "auto:4,7" self._fuzziness_lo, self._fuzziness_hi = 4, 7 self._min_token_size_for_fuzziness = self._fuzziness_lo # self.set_fuzziness_threshold(fuzziness=(self._fuzziness_lo, self._fuzziness_hi)) # defaults for non-auto mode self.set_fuzziness_threshold(fuzziness=1) self._min_token_size_for_fuzziness = 4 self.db = DataStore()
def preprocess(dbPath): ''' This is a preprocess module ''' logging = DefaultLogger() if not os.path.exists(dbPath): logging.debug('PreProcess: can\'t find database at path') return datastore = DataStore(dbPath) loopcount = 0 while True: sleep(5) if loopcount % 10 == 0: logging.debug('PreProcess is alive') loopcount += 1 data = datastore.recordsForHashing() for record in data: logging.debug(record) key_id = record.id filePath = record.fileName if not os.path.exists(filePath): logging.debug( 'PreProcess: Will update record status as the file no longer exists' ) datastore.updateRecordAsMissingWithID(key_id) continue try: logging.debug('PreProcess: locking file to calculate hash...') ##UPDATE HASH OPERATION START HERE startTime = datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S") datastore.updateRecordWithHashStart(startTime, key_id) fileToHash = open(filePath, 'rb') portalocker.lock(fileToHash, portalocker.LOCK_SH) hashString = "NO_OP" #hashForFile(fileToHash) endTime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") fileToHash.close() logging.debug('PreProcess: unlocking file...') logging.debug( 'PreProcess: Will update record status with Hash string and times' ) datastore.updateRecordWithHashForStartTimeAndEndTime( hashString, startTime, endTime, key_id) except Exception as e: info = 'PreProcess: There was an error when calculating the hash for file: ' + os.path.basename( filePath) + ' ' + e.message sendFailureEmail(info) logging.error(e.message)
class Session: def __init__(self): """Constructor for the session class to store information regarding the session table Args: None Returns: The Session object """ self._databaseName = "dynamodb" self._tableName = "Sessions" self._partitionKeyName = "PatientId" self._sortKeyName = "SessionId" self._indexName = "hcpIndex" self._indexPartitionKeyName = "HealthCareProfessionalId" self._indexSortKeyName = "SessionId" self._sessionDataStore = DataStore(self._databaseName, self._tableName, self._partitionKeyName, self._sortKeyName, self._indexName, self._indexPartitionKeyName, self._indexSortKeyName) def createSession(self, info): """Create the session and store the data into database Args: info(dict): information regarding the session (patient id, session id, health care professional id, session name etc.) Returns: None """ return self._sessionDataStore.save(info) def requestSession(self, partitionKey, sortKey, indexPartitionKey): """Request and get the specific session(s) from the database. If both partition key and sort key are provided, return that single specific session. if both index partition key and sort key are provided, return that single specific session. if only partition key is provided, return sessions with that partition key. if onyl index partition key is provided, return sessions with that index partition key. Args: partitionKey(str): partition key value of the session table sortKey(str): sort key value of the session table indexPartitionKey(str): index partition key value of the session table Returns: List of session(s) info """ if partitionKey and sortKey: return self._sessionDataStore.queryByBothKeys(partitionKey, sortKey) elif indexPartitionKey and sortKey: return self._sessionDataStore.queryByIndexBothKeys(indexPartitionKey, sortKey) elif partitionKey: return self._sessionDataStore.queryByPartitionKey(partitionKey) elif indexPartitionKey: return self._sessionDataStore.queryByIndexPartitionKey(indexPartitionKey) return {"Items": [], "status": "BAD", "message": "Wrong Way to Search."}
def update_poi_test(id): store = DataStore() poi = PointOfInterest.get_test_item(id) poi.name = "t_name_toUpdate" poi.category = "t_category_toUpdate" poi.creator = "t_creator_toUpdate" poi.description = "_description_toUpdate" poi.latitude = 20.00000 poi.longitude = 10.00000 poi = store.update_poi(poi) return to_json(poi)
def test_open_data_store_raises_FileNotFound(self): with self.assertRaisesRegex( FileNotFoundError, "The root of the data store: 'testing/data/does_not_exist' does not exist" ): ds = DataStore('testing/data/does_not_exist') with self.assertRaisesRegex( FileNotFoundError, "The root of the data store: 'testing/data/xxx' is not a directory" ): ds = DataStore('testing/data/xxx')
def setUp(self): self.datastore = DataStore(initial_dataset=[ { 'id': 1, 'name': 'Nemeziz', }, { 'id': 2, 'name': 'Copa Mundial', }, ], )
def initialize_connections(self): if len(self.socket_table) < TaskHandler.PEER_REPICK: if DBGMODE : print("initialize_connections [start]") DataStore.mutex.acquire() self.datastore = DataStore() self.peers = self.datastore.get_highest_rating(10) for peer in self.peers: if peer.id not in self.socket_table and (peer.ip != self.remote_ip or peer.port !=self.remote_port): self.socket_table[peer.id] = (peer.ip, int(peer.port)) self.datastore.close() DataStore.mutex.release()
def setUp(self): db = sqlite3.connect(":memory:") # Populate the datastore self.ds = DataStore(db) data = [1, 2, 3, 4, 5] self.ds.segments.addSegment(0x1234, len(data), "ROM", data) self.seg = self.ds.segments.__iter__().next() del self.ds # Reload the datastore from the db self.ds = DataStore(db)
def test_save_collision(self): table = self.create_session_table() d = DataStore('dynamodb', 'Sessions', 'PatientId', 'SessionId') d.save(SESSION_INFO_1) response = table.scan() self.assertEqual(1, len(response['Items'])) self.assertEqual('SessionName', response['Items'][0]['SessionName']) response = d.save(SESSION_INFO_1_SAMEKEY) self.assertEqual('OK', response['status']) response = table.scan() self.assertEqual(1, len(response['Items'])) self.assertEqual('NO', response['Items'][0]['SessionName'])
def __init__(self): try: self.auth = OAuthHandler(Secretes._twitterConsumerApiKey, Secretes._twitterConsumerApiSecret) self.auth.secure = True self.auth.set_access_token(Secretes._twitterAccessToken, Secretes._twitterAccessTokenSecret) self.api = API(self.auth) self.dataStore = DataStore() except BaseException as e: print("Error in main()", e) self.tweetCount = 0
def __init__(self): try: logging.basicConfig(filename=GlobalVariables._logFileName) self.auth = OAuthHandler(Secretes._twitterConsumerApiKey, Secretes._twitterConsumerApiSecret) self.auth.secure = True self.auth.set_access_token(Secretes._twitterAccessToken, Secretes._twitterAccessTokenSecret) self.api = API(self.auth) self.dataStore = DataStore() except BaseException as e: print("Error in main()", e) self.tweetCount = 0
def test3(self): con = make_dbcon() ds = DataStore(con) users = ds.collection("users") users.add_index("email") uuid = users.save({'email': '*****@*****.**', 'name':'John Doe!'}) self.assertIsNotNone(users.index("email").find("*****@*****.**")) users.delete(uuid) self.assertEqual(len(users.index("email").find("*****@*****.**")), 0)
def start(): fileConfig('conf/log.conf') logging.getLogger('garage').log(logging.DEBUG, 'Log setup complete') logging.getLogger('garage').log(logging.DEBUG, 'Initializing datastore ') db = DataStore(setup='true') db.shutdown() logging.getLogger('garage').log(logging.DEBUG, 'Complete') #butler = None butler = Butler() web.start(butler)
def get_tasks(): u = request.form['url'].lower() url = Utilities.get_shortened_url(u) url_3 = Utilities.get_shortened_url(u,3) return_only_parent = False # If url is same as parent url, return everything just for parent # Dont redundantly return for parent and itself if url == url_3 or url+'/' == url_3: return_only_parent = True ds = DataStore() if not return_only_parent: all_urls = Utilities.modify_url(url) print all_urls # If the same url is also a parent url, return all results of parent . # And skip individual url results for url in all_urls: result = ds.fetch(url) if result == False: print " Tried for url " + url else: x = {"result":result} return jsonify(x) # If for our exact url and its modifications , nothing got returned outer_url = "parent::" + Utilities.get_shortened_url(url,3) print outer_url result = ds.fetch_all_from_parent(outer_url) if result : x = {"result":result} return jsonify(x) else: if outer_url[-1] == '/': result = ds.fetch_all_from_parent(outer_url[:-1]) else: result = ds.fetch_all_from_parent(outer_url + '/') if result : x = {"result":result} return jsonify(x) # If there is still nothing to show return 'No Response'
def test2(self): con = make_dbcon() ds = DataStore(con) col = ds.collection("users") col.add_index("email") uuid = col.save({'email': '*****@*****.**', 'name':'My Name!'}) i = col.index('email') self.assertIsNone(i.findone('*****@*****.**')) doc = i.findone('*****@*****.**') self.assertEqual(doc['uuid'], uuid)
def get_pois(): store = DataStore() query_dic = dict() use_query = False for k in PointOfInterest().__dict__.keys(): v = request.args.get(k, None) if v is not None: query_dic[k] = v use_query = True if use_query: pois = store.get_poi_filtered(query_dic) else: pois = store.get_all_poi() return to_json(pois)
def handle_put(seq,key, value): """Return a tuple containing True and the message to send back to the client.""" if key not in POROCESSING: POROCESSING.append(key) ds = DataStore(key,value) if ds.put(seq): POROCESSING.remove(key) return (True, 'Key [{}] set to [{}]'.format(key, value)) else: ds.roll_back(seq) POROCESSING.remove(key) return (False, 'Could Not be added')
def handle_delete(seq,key): """Return a tuple containing True if the key could be deleted and the message to send back to the client. use datastore.get and then before doing doing datastore.delete """ ds = DataStore() if ds.get(key): POROCESSING.append(key) if ds.delete(seq,key): POROCESSING.remove(key) return (True,'Done') else: ds.roll_back(seq) POROCESSING.remove(key) return (False,'ERROR: Key [{}] not found and could not be deleted'.format(key))
def test4(self): con = make_dbcon() ds = DataStore(con) users = ds.collection("users") users.add_index("email") uuid = users.save({'email': '*****@*****.**', 'name':'John Doe!'}) doc = users.load(uuid) self.assertIsNotNone(users.index.email.findone('*****@*****.**')) del doc['email'] users.save(doc) self.assertIsNone(users.index.email.findone('*****@*****.**'))
def test2(self): con = make_dbcon() ds = DataStore(con) col = ds.collection("users") uuid = col.save({ 'email':'*****@*****.**', 'name':'Roel Gerrits' }) doc = col.load(uuid) self.assertDictEqual(doc, { 'uuid': uuid, 'email':'*****@*****.**', 'name':'Roel Gerrits' })
def test3(self): con = make_dbcon() ds = DataStore(con) col = ds.collection("users") uuid = col.save({ 'email':'*****@*****.**', 'name':'Roel Gerrits' }) self.assertTrue(col.delete(uuid)) self.assertFalse(col.delete(uuid)) doc = col.load(uuid) self.assertIsNone(doc)
def preprocess(dbPath): ''' This is a preprocess module ''' logging = DefaultLogger() if not os.path.exists(dbPath): logging.debug('PreProcess: can\'t find database at path') return datastore = DataStore(dbPath) loopcount = 0 while True: sleep(5) if loopcount % 10 == 0: logging.debug('PreProcess is alive') loopcount += 1 data = datastore.recordsForHashing() for record in data: logging.debug(record) key_id = record.id filePath = record.fileName if not os.path.exists(filePath): logging.debug('PreProcess: Will update record status as the file no longer exists') datastore.updateRecordAsMissingWithID(key_id) continue try: logging.debug('PreProcess: locking file to calculate hash...') ##UPDATE HASH OPERATION START HERE startTime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") datastore.updateRecordWithHashStart(startTime, key_id) fileToHash = open(filePath, 'rb') portalocker.lock(fileToHash, portalocker.LOCK_SH) hashString = "NO_OP"#hashForFile(fileToHash) endTime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") fileToHash.close() logging.debug('PreProcess: unlocking file...') logging.debug('PreProcess: Will update record status with Hash string and times') datastore.updateRecordWithHashForStartTimeAndEndTime(hashString, startTime, endTime, key_id) except Exception as e: info = 'PreProcess: There was an error when calculating the hash for file: ' + os.path.basename(filePath) + ' ' + e.message sendFailureEmail(info) logging.error(e.message)
def test1(self): con = make_dbcon() ds = DataStore(con) col = ds.collection("users") col.save({ 'uuid': 'f8e5519ee55811e2839a0024219c6b57', 'email':'*****@*****.**', 'name':'Roel Gerrits' }) doc = col.load('f8e5519ee55811e2839a0024219c6b57') self.assertDictEqual(doc, { 'uuid': 'f8e5519ee55811e2839a0024219c6b57', 'email':'*****@*****.**', 'name':'Roel Gerrits' })
def __init__(self, filename): """Initializes a Text.""" assert filename self.filename = filename self.text = self.get_text_from_txt_file() if DataStore.is_to_be_computed(filename): database = DataStore(filename, self.__compute_ds_keys_values()) else: database = DataStore(filename) self.ds = database
def update_database(self, peers): DataStore.mutex.acquire() database_conn = DataStore() current_nodes = database_conn.get_all_peers() new_list = [] for peer in peers: overlay = False for node in current_nodes: if node.ip == peer["ip"] and node.port == peer["port"]: if node.rating < 30: node.rating = 30 database_conn.update(node) overlay = True if overlay == False: new_list.append(PeerNode(0, peer["id"], peer["ip"], peer["port"], 30)) database_conn.insert_peers(new_list) database_conn.close() DataStore.mutex.release()
def __init__(self): super(CalendarPlugin, self).__init__() self.first_day = self.last_day = self.numdays = None builder = Gtk.Builder() builder.add_from_file("calendar_view.glade") handlers = { "on_window_destroy": Gtk.main_quit, "on_today_clicked": self.on_today_clicked, "on_combobox_changed": self.on_combobox_changed, "on_add_clicked": self.on_add_clicked, "on_edit_clicked": self.on_edit_clicked, "on_remove_clicked": self.on_remove_clicked, "on_next_clicked": self.on_next_clicked, "on_previous_clicked": self.on_previous_clicked, } builder.connect_signals(handlers) self.window = builder.get_object("window") self.window.__init__() self.window.set_title("GTG - Calendar View") self.window.connect("destroy", Gtk.main_quit) # DataStore object self.ds = DataStore() self.req = Requester(self.ds) self.ds.populate() # hard-coded tasks self.today_button = builder.get_object("today") self.header = builder.get_object("header") self.controller = Controller(self, self.req) vbox = builder.get_object("vbox") vbox.add(self.controller) vbox.reorder_child(self.controller, 1) self.current_view = None self.combobox = builder.get_object("combobox") self.combobox.set_active(2) self.statusbar = builder.get_object("statusbar") self.window.show_all()
"reference:": reference, "title": documents[0]["title"].decode("utf-8"), "date": datestring, "attachment": solr_attachments.values(), "session": solr_sessions.values(), "committee": solr_committees.values(), "street": unique(solr_streets.keys()), "person": solr_people.values(), "content": solr_body, #'location': positions_for_streetnames(unique(solr_streets.keys())) } if __name__ == "__main__": s = solr.SolrConnection(config.SOLR_URL) db = DataStore(config.DBNAME, config.DBHOST, config.DBUSER, config.DBPASS) parser = OptionParser() parser.add_option( "-s", "--sample", dest="sample", default=1, help="z.B. die Zahl 10 um nur jedes zehnte Dokument zu importieren. Beschleunigt den Import beim Entwickeln.", ) parser.add_option( "-v", "--verbose", dest="verbose", default=False, action="store_true", help="Aktiviert die detailliertere Ausgabe (Dokumententitel etc.).",
def on_data(self,data): try: data = json.loads(data) newd = {} # Get Tweet tweet = Utilities.clean_tweet(data['text']) for key in self.recent_tweets: #print Utilities.similarity(key,tweet) if Utilities.similarity(key,tweet) > 70: return ''' if tweet in self.recent_tweets: return else: ''' if len(self.recent_tweets) > 50: self.recent_tweets.popitem(last=False) self.recent_tweets[tweet] = True #print tweet # Get Redirected url try: url_name = Utilities.get_redirected_url(str(data['entities']['urls'][0]['expanded_url'])) except: return raise BaseException("Url for tweet did not exist") # Get shortened url for key --> Upto 5th '/' or entire address (whichever is shorter) url_name = Utilities.get_shortened_url(url_name).lower() #Get timestamp timestamp = str(data['created_at']) # Verify authenticity of website by checking if it has the word coupon # If it does , assume it is not a vendor site. Maybe blog, maybe coupon site try: Utilities.check_url_validity(url_name) except: return raise BaseException("Url was not a valid site") with open("x.txt","a") as f: f.write(tweet + '\n') f.write("--------------------" + '\n') # Code to extract important information from this tweet #self.tweets += 1 #print tweet #print "Tweet Number : " + str(self.tweets) e = Extraction() code,date = e.extract_all(tweet) if not code: #print " --------------- " return raise BaseException("Did not have coupon code information") if not date : date = 183600 else : self.tweets_with_dates += 1 self.total_expiry_time += date self.exp_time.append(date/3600) print self.tweets_with_dates, int(numpy.median(self.exp_time)) #print date #print self.tweets_with_dates print tweet #print " ----------------------------------- " #print "Tweet : ", #print "Url : ", #print url_name #print "Date : " #print "Coupons : " + str(self.tweets_with_coupons) #print "Dates : " + str(self.tweets_with_dates) #print "Total Expiry Time :" + str(self.total_expiry_time/3600) + "hours" #print "Avg Expiry Time :" + str((self.total_expiry_time/(self.tweets_with_dates+1))/3600) + "hours" print '--------------------------------------' #print "CODE : " + code key = url_name + ':::' + code #print "KEY : " + key #print "Tweet : " #print tweet #print "Url : ", #print url_name #print " ----------------------------------- " ds = DataStore() #print url_name,code,date #get outer url - url uptil 3 '/'s . eg - http://www.etsy.com/ outer_url = "parent::"+Utilities.get_shortened_url(url_name,3) ds.insert(key,url_name,code,tweet,date,outer_url) #print '-----------------------' return True except BaseException as e: if str(e) != "'text'": #print " *************** " + str(e) + " *************** " #print "----------------------------------------" pass time.sleep(1)
def handle_keys(): ds = DataStore() keys = ds.mykeys() keys = [x[0] for x in keys] keys_n = '-'.join(str(x) for x in keys) return (True,keys_n)