Beispiel #1
0
    def __init__(self, entity_name=None, source_language_script=ENGLISH_LANG, translation_enabled=False):
        """
        Initializes a TextDetector object with given entity_name

        Args:
            entity_name: A string by which the detected substrings that correspond to text entities would be replaced
                         with on calling detect_entity()
            source_language_script: ISO 639 code for language of entities to be detected by the instance of this class
            translation_enabled: True if messages needs to be translated in case detector does not support a
                                 particular language, else False
        """
        # assigning values to superclass attributes
        self._supported_languages = [ENGLISH_LANG, HINDI_LANG]
        super(TextDetector, self).__init__(source_language_script, translation_enabled)

        self.text = None
        self.text_dict = {}
        self.tagged_text = None
        self.text_entity_values = []
        self.original_texts = []
        self.processed_text = None
        self.entity_name = entity_name
        self.tag = '__' + self.entity_name + '__'

        # defaults for auto mode
        self._fuzziness = "auto:4,7"
        self._fuzziness_lo, self._fuzziness_hi = 4, 7
        self._min_token_size_for_fuzziness = self._fuzziness_lo
        # self.set_fuzziness_threshold(fuzziness=(self._fuzziness_lo, self._fuzziness_hi))

        # defaults for non-auto mode
        self.set_fuzziness_threshold(fuzziness=1)
        self._min_token_size_for_fuzziness = 4

        self.db = DataStore()
 def test_get_references(self):
     ds = DataStore('testing/data/reference_types')
     with self.assertRaises(NotAManagedType):
         ds.type('XXX')
     references = ds.type('Type1')
     self.assertTrue(hasattr(references, 'get'),
                     'References does not include a get attribute')
     self.assertTrue(callable(getattr(references, 'get')),
                     'References.get is not callable')
     self.assertTrue(hasattr(references, 'add'),
                     'References does not include a add attribute')
     self.assertTrue(callable(getattr(references, 'add')),
                     'References.add is not callable')
     self.assertTrue(hasattr(references, 'update'),
                     'References does not include a update attribute')
     self.assertTrue(callable(getattr(references, 'update')),
                     'References.update is not callable')
     self.assertTrue(hasattr(references, 'filter'),
                     'References does not include a filter attribute')
     self.assertTrue(callable(getattr(references, 'filter')),
                     'References.filter is not callable')
     self.assertTrue(hasattr(references, 'delete'),
                     'References does not include a delete attribute')
     self.assertTrue(callable(getattr(references, 'delete')),
                     'References.delete is not callable')
class EncryptorWatcher(LoggingEventHandler):
    '''
    This class enters all file 'created' events to a database pointed to by dbFolder
    '''
    def __init__(self, pathStructure, dbFolder):
        super(LoggingEventHandler, self).__init__()
        self.pathStructure = pathStructure
        self.dataStore = DataStore(dbFolder) 

    def on_modified(self, event):
        path = os.path.join(self.pathStructure['inBox'], event.src_path)
        logging.debug("encryptorWatch on_modified file")
        info = "Modified: " +  event.src_path + " " + str(os.path.getsize(path))
        logging.debug(info)

    def on_created(self, event):
        path = os.path.join(self.pathStructure['inBox'], event.src_path)
        
        if os.path.isdir(os.path.abspath(event.src_path)):
            logging.debug('WatchProcess: Folder Encryption is not supported.')
            return

        self.dataStore.addFilePathToDataBaseStoreWithType(os.path.abspath(event.src_path), self.pathStructure['watchType'], self.pathStructure['name'])

        info = "Created: " +  event.src_path + " " + str(os.path.getsize(path))
        logging.debug("encryptorWatch on_created file")
        logging.debug(info)
 def test_filter_reference(self):
     ds = DataStore('testing/data/reference_types')
     results = ds.type('Type1').filter(name='Item_1')
     self.assertEqual(1, len(results))
     self.assertEqual('1', results[0].ref)
     self.assertEqual('Item_1', results[0].name)
     self.assertEqual('Item 1', results[0].description)
Beispiel #5
0
 def test_save_empty(self):
     table = self.create_session_table()
     d = DataStore('dynamodb', 'Sessions', 'PatientId', 'SessionId')
     response = d.save(SESSION_INFO_EMPTY)
     self.assertEqual('BAD', response['status'])
     response = table.scan()
     self.assertEqual(0, len(response['Items']))
Beispiel #6
0
def handle(config):
	try:
                flagcheck=True
                logger.info("calling jmx for metrics")
		#check for jmx hosts file
		#TODO add code for handling metrics from multiple JMX hosts	
		#
		#JAVA(libjvm='./lib/jmx/libjvm.so')
		#JAVA()
		jpype.attachThreadToJVM()
		jmx=JMX(host='96.119.153.107',port=9999)
		DS=DataStore()
                for condition in config.get('if').get('jmx'):
			baseline=DS.getbaseline(condition)
    			current=jmx.get_attr(condition.get('object'),condition.get('type'),condition.get('attribute'))
			logger.debug(current)
			logger.debug(str(current) + condition.get('operator') + repr(baseline))
			out=eval(str(current) + condition.get('operator') + repr(baseline))
			if not bool(out):
                                flagcheck=False
                                break
			DS.setbaseline(current.floatValue(),baseline,condition)
    		del jmx
                return flagcheck
        except Exception,e:
		print "in exception"
		print e
                logger.error(e)
                return False
Beispiel #7
0
class App(tk.Frame):
    def __init__(self, parent, *args, **kwargs):
        tk.Frame.__init__(self, parent, *args, **kwargs)
        self.parent = parent
        
        self.datastore = DataStore()
        
        self.plclink = PLCLink()
        self.varlist = getvars()
        
        # Two frames, display frame and control frame
        displayfrm = PlotWidget(self, relief=RAISED, borderwidth=1)
        displayfrm.pack(fill=BOTH, expand=1)

        ctrlfrm = ControlWidget(self)
        ctrlfrm.pack(fill=BOTH, expand=1)
        
    def logger(self):
        try:
            data = self.plclink.getdata()
            self.datastore.insert(data)
        except:
            print("Logging failed on {}".format(strftime("%b %d %Y %H:%M:%S", 
                  localtime())))
        finally:
            # Log again after specified interval
            self.master.after(loginterval, self.logger)
    def setplot(self):
        PlotConfigurator(self, self.varlist)
Beispiel #8
0
    def __init__(self):
        self.state = State.Init
        self.config = {
            'hover_height':
            1.0,

            # States for which movement.fix_hover() will NOT be called (to make sure the drone is at `hover_height`)
            'exclude_from_fix_hover': [
                State.Init,
                State.Takeoff,
                State.Land,
                State.Done,
            ],

            # Radius in meters around a blacklisted goal that the robot will ignore
            'blacklisted_goal_radius':
            2.0
        }

        self.store = DataStore()
        self.movement = MovementHandler(core=self, datastore=self.store)
        self.actions = Actions(core=self,
                               datastore=self.store,
                               movement_handler=self.movement)
        self.planner = Planner(core=self,
                               datastore=self.store,
                               movement_handler=self.movement)

        # Aux files
        self.temp_data = {}
        self.last_goal = None
Beispiel #9
0
class singleFileWatcher(LoggingEventHandler):
    '''
    This class enters all file 'created' events to a database pointed to by dbFolder
    '''
    def __init__(self, pathStructure, dbFolder):
        super(LoggingEventHandler, self).__init__()
        self.pathStructure = pathStructure
        self.dataStore = DataStore(dbFolder)

    def on_created(self, event):

        for ignoreFile in ['.DS_Store', 'Thumbs.db']:
            if ignoreFile in os.path.abspath(event.src_path):
                info = 'File ignored: ' + os.path.abspath(event.src_path)
                logging.debug(info)
                return

        info = 'On created: ' + os.path.abspath(event.src_path)
        logging.debug(info)

        if os.path.isdir(os.path.abspath(event.src_path)):
            info = 'Directory analysis is not available'
            logging.debug(info)
            return

        self.dataStore.addFileToDatabase(os.path.abspath(event.src_path))
        info = 'adding ' + event.src_path + ' to the database'
        logging.debug(info)
Beispiel #10
0
 def close(self):
     self.commitStockBatch()
     self.commitBatch()
     self.conn.commit()
     self.conn.close()
     self.conn = None
     DataStore.close(self)
Beispiel #11
0
class singleFileWatcher(LoggingEventHandler):
    '''
    This class enters all file 'created' events to a database pointed to by dbFolder
    '''
    def __init__(self, pathStructure, dbFolder):
        super(LoggingEventHandler, self).__init__()
        self.pathStructure = pathStructure
        self.dataStore = DataStore(dbFolder) 

    def on_created(self, event):
        
        for ignoreFile in ['.DS_Store', 'Thumbs.db']:
            if ignoreFile in os.path.abspath(event.src_path):
                info = 'File ignored: ' +  os.path.abspath(event.src_path)
                logging.debug(info)
                return

        info = 'On created: ' +  os.path.abspath(event.src_path)
        logging.debug(info)

        if os.path.isdir(os.path.abspath(event.src_path)):
            info = 'Directory analysis is not available'
            logging.debug(info)
            return

        self.dataStore.addFileToDatabase(os.path.abspath(event.src_path))
        info = 'adding ' + event.src_path + ' to the database'
        logging.debug(info)
Beispiel #12
0
    def getcompany_info(self, name, url):
        logger = Logger(logname='error.log', logger="58com").getlog()
        ds = DataStore()
        try:
            company_text = []
            html = proxy.proxy_request(url)
            soup = BeautifulSoup(html, 'html.parser')
            tag = soup.find(class_="basicMsg")
            ul = tag.find("ul")
            li_tags = ul.find_all(name='li')
            strinfo = re.compile('\s')
            for li in li_tags:
                txt = strinfo.sub('', li.get_text())
                company_text.append(txt.split(':')[1])
            #获取工商信息
            #gongshang_info = tianyan.tianyan_search(name)
            #gongshang_info = ','.join(gongshang_info)
            ds.insert_database(name, company_text)

        except urllib.error.URLError as e:
            if hasattr(e, "code"):
                print(e.code)
            if hasattr(e, "reason"):
                print(e.reason)
            logger.error("Get company info fail, company name: %s, url: %s",
                         name, url)  #记录解析失败的公司和url
        except Exception as e:
            print("exception:" + str(e))
            sleep(1)
    def handle(self, *args, **options):
        entity_data_directory_path = None
        csv_file_paths = None
        if ('entity_data_directory_path' in options and options['entity_data_directory_path']) or \
                ('csv_file_paths' in options and options['csv_file_paths']):

            if 'entity_data_directory_path' in options and options['entity_data_directory_path']:
                entity_data_directory_path = options['entity_data_directory_path']
                if not os.path.exists(entity_data_directory_path):
                    entity_data_directory_path = None

            if 'csv_file_paths' in options and options['csv_file_paths']:
                csv_file_paths = options['csv_file_paths'].split(',')
                csv_file_paths = [csv_file_path for csv_file_path in csv_file_paths if csv_file_path and
                                  csv_file_path.endswith('.csv')]
            db = DataStore()
            db.repopulate(entity_data_directory_path=entity_data_directory_path, csv_file_paths=csv_file_paths)
            if entity_data_directory_path:
                self.stdout.write(
                    'Successfully Repopulated entity data from csv files at "%s"' % entity_data_directory_path)
            if csv_file_paths:
                self.stdout.write('Successfully Repopulated entity data from %d other csv file(s)"'
                                  % len(csv_file_paths))
        else:
            self.stdout.write(self.style.ERROR('argument --entity_data_directory_path or --csv_file_paths required'))
 def test_add_reference_raises_DuplicateReference(self):
     ds = DataStore('testing/data/reference_types')
     type1 = ds.type('Type1').get("1")
     with self.assertRaises(Type1.DuplicateReference):
         x = ds.type('Type1').add(type1)
     with self.assertRaises(Type1.DuplicateReference):
         x = ds.type('Type1').add({}, ref="1")
 def test_delete_reference_raises_DoesNotExist(self):
     ds = DataStore('testing/data/reference_types')
     typex = Type1(ref='X', name='Type_X', desc='Type X')
     with self.assertRaises(Type1.DoesNotExist):
         x = ds.type('Type1').delete(typex)
     with self.assertRaises(Type1.DoesNotExist):
         x = ds.type('Type1').delete({}, ref="X")
Beispiel #16
0
    def __init__(self, entity_name=None):
        """
        Initializes a TextDetector object with given entity_name

        Args:
            entity_name: A string by which the detected substrings that correspond to text entities would be replaced
                         with on calling detect_entity()
        """
        self.text = None
        self.regx_to_process = Regex([(r'[\'\/]', r'')])
        self.text_dict = {}
        self.tagged_text = None
        self.text_entity = []
        self.original_text_entity = []
        self.processed_text = None
        self.entity_name = entity_name
        self.tag = '__' + self.entity_name + '__'

        # defaults for auto mode
        self._fuzziness = "auto:4,7"
        self._fuzziness_lo, self._fuzziness_hi = 4, 7
        self._min_token_size_for_fuzziness = self._fuzziness_lo
        # self.set_fuzziness_threshold(fuzziness=(self._fuzziness_lo, self._fuzziness_hi))

        # defaults for non-auto mode
        self.set_fuzziness_threshold(fuzziness=1)
        self._min_token_size_for_fuzziness = 4

        self.db = DataStore()
Beispiel #17
0
def main():
	"""Parse the command line arguments, expecting one of the following formats:
		-) (-i ChannelID | -u Username) (add | check | remove)
		-) check | list
	and perform the appropriate action
	"""
	parser = get_parser()
	args = parser.parse_args()

	youtube = YouTube()
	store = DataStore('%s-data.sqlite3' % sys.argv[0], 'schema.sql')

	channel = None
	if args.username is not None:
		channel = youtube.get_channel_by_username(args.username)
	elif args.id is not None:
		channel = youtube.get_channel_by_id(args.id)

	if args.action == 'add':
		store.store_channel(channel)
	elif args.action == 'remove':
		store.remove_channel(channel)
	elif args.action == 'list':
		data = []
		for item in store.get_channels():
			data.append([
				item['id'],
				item['title'],
				arrow.get(item['added_on']).humanize(),
				arrow.get(item['last_checked']).humanize()
			])

		pretty_print(['ID', 'Title', 'Added', 'Last Checked'], data)
	elif args.action == 'check':
		# If the user passed a specific channel, check for new uploads
		# otherwhise check for uploads from every previously added channel
		channels = []
		if channel is not None:
			channels.append(store.get_channel_by_id(channel['id']))
		else:
			channels = store.get_channels()

		data = []
		to_check = dict()
		for channel_item in channels:
			to_check[channel_item['id']] = channel_item['last_checked']

		uploads = youtube.get_uploads(to_check)
		for upload in uploads:
			data.append([
				upload['channel_title'],
				upload['title'],
				arrow.get(upload['published_at']).humanize(),
				'https://youtube.com/watch?v=%s' % (upload['id'], )
			])

		pretty_print(['Channel', 'Title', 'Published', 'Link'], data)

		for channel_id in to_check.keys():
			store.update_last_checked(channel_id)
Beispiel #18
0
class Achievements:
    __borg_state = {}
    __achievements_id = "account_achievements"

    def __init__(self, config):
        self.__dict__ = self.__borg_state
        self.config = config
        self.api = GW2_API(config)
        self.ds = DataStore(config)
        self.render = Render(config)

    def _get_current(self):
        """Get the current achievements for the character."""

        cheeves = self.api.get("account/achievements")
        cheeves_by_id = {}
        for cheeve in cheeves:
            cheeves_by_id[cheeve.id] = cheeve

        return cheeves_by_id

    def _get_new_unlock_cheeves(self, old_cheeves, new_cheeves):
        """Given a dict of old and new Achievements, find those that are
        newly unlocked.

        Returns a tuple of:
            (unlocks, newness) where
                    -unlocks is newly completed cheeves
                    -newness is new added cheeves
        """

        unlocks = []
        newness = []
        for cheeve in new_cheeves:
            if cheeve['id'] not in old_cheeves:
                newness.append(cheeve)
            elif cheeve['done'] != old_cheeves[cheeve['id']]['done']:
                unlocks.append(cheeve)
        return (unlocks, newness)

    def _get_new_progress_cheeves(self, old_cheeves, new_cheeves):
        """Given a dict of old and new Achievements, find those that have
        new progress on them."""

        new_prog = []
        for cheeve in new_cheeves:
            if cheeve.get('current', 0) != \
                    old_cheeves[cheeve['id']].get('current', 0):
                new_prog.append(cheeve)

    def update(self, cheeves=None):
        """Will update the datastore with the current cheeves. Intended to be
        called once per day, per week, per cycle (whatever).

        If 'cheeves' is ommitted, will get the current cheevese via API"""

        if cheeves is None:
            cheeves = self._get_current()

        self.ds.put(self.__achievements_id, cheeves)
Beispiel #19
0
    def __init__(self, prodj):
        super().__init__()
        self.prodj = prodj
        self.queue = Queue()

        self.pdb_enabled = True
        self.pdb = PDBProvider(prodj)

        self.dbc_enabled = True
        self.dbc = DBClient(prodj)

        # db queries seem to work if we submit player number 0 everywhere (NOTE: this seems to work only if less than 4 players are on the network)
        # however, this messes up rendering on the players sometimes (i.e. when querying metadata and player has browser opened)
        # alternatively, we can use a player number from 1 to 4 without rendering issues, but then only max. 3 real players can be used
        self.own_player_number = 0
        self.request_retry_count = 3

        self.metadata_store = DataStore(
        )  # map of player_number,slot,track_id: metadata
        self.artwork_store = DataStore(
        )  # map of player_number,slot,artwork_id: artwork_data
        self.waveform_store = DataStore(
        )  # map of player_number,slot,track_id: waveform_data
        self.preview_waveform_store = DataStore(
        )  # map of player_number,slot,track_id: preview_waveform_data
        self.beatgrid_store = DataStore(
        )  # map of player_number,slot,track_id: beatgrid_data
Beispiel #20
0
def perform_search(search_str,results_collector):

    base_url = 'http://sfbay.craigslist.org'

    cl_html_results = requests.get(search_str)
    soup = BeautifulSoup(cl_html_results.text, 'html.parser')

    ds = DataStore(storetype='sql')

    for result in soup.find_all(attrs={"data-pid": re.compile('\d+')}):

        link_title = result.find(id='titletextonly')

        if link_title is None:
            # print "Cannot find title for entry %s", result
            next
        else:
            datapid = result.attrs['data-pid']
            link_title_text = link_title.text

            link = '{0}{1}'.format(base_url, result.find('a').attrs['href'])

            #print "debug: {0} | {1} | {2}".format(datapid, link_title_text, link)

            ds.save_entry(datapid=datapid, title=link_title_text, url=link)

    for i in ds.new_listings:
        results_collector.append(i)
Beispiel #21
0
 def handle(self, *args, **options):
     db = DataStore()
     if db.exists():
         db.delete()
         self.stdout.write('Successfully deleted DataStore structure and data.')
     else:
         self.stdout.write('DataStore structure not created. Skipping delete action')
Beispiel #22
0
 def handle(self, *args, **options):
     if 'entity_name' in options and options['entity_name']:
         entity_name = options['entity_name']
         db = DataStore()
         db.delete_entity(entity_name=entity_name)
         self.stdout.write('Successfully deleted entity data for "%s"' % entity_name)
     else:
         self.stdout.write(self.style.ERROR('argument --entity_name required'))
Beispiel #23
0
 def test1(self):
     con = make_dbcon()
     ds = DataStore(con)
     col = ds.collection("users")
     
     i = Index(con, col, 'email')
     
     self.assertEqual(i.name(), 'email')
Beispiel #24
0
def unknown_negative():
    global ds
    _ds = DataStore('/tmp/pood/ds')
    paths = _ds.fetch(1).all().paths()
    if len(paths) > 0:
        path = paths[0]
        ds.store(0).tile(Image.open(open(path, mode='rb')), 100, size=(16, 16))
        os.unlink(path)
    return redirect('/')
def handle_get(key):
    """Return a tuple containing True if the key exists and the message
    to send back to the client."""
    if key not in POROCESSING:
        ds = DataStore()
        data = ds.get(key)
        if data:
            return(True, (data[0],data[1]))
    return(False, 'ERROR: Key [{}] not found'.format(key))
Beispiel #26
0
def update_poi(id):
    store = DataStore()
    poi = PointOfInterest.from_request(request)
    poi.id = id
    valid = poi.validate()
    if len(valid) == 0:
        poi = store.update_poi(poi)
        return to_json(poi)
    else:
        return to_json(valid)
 def handle(self, *args, **options):
     db = DataStore()
     if db.exists():
         self.stdout.write(
             'DataStore structure already created, skipping this action. '
             'If you want to recreate it, run python manage.py delete_datastore first.'
         )
     else:
         db.create()
         self.stdout.write('Successfully created DataStore structure')
Beispiel #28
0
 def test_save(self):
     table = self.create_session_table()
     d = DataStore('dynamodb', 'Sessions', 'PatientId', 'SessionId')
     response = d.save(SESSION_INFO_1)
     self.assertEqual('OK', response['status'])
     response = table.scan()
     item = response['Items'][0]
     self.assertTrue(
         'ComprehendS3Path' in item[DATASTORE_COLUMN_COMPREHEND_S3_PATH])
     self.assertEqual('p-1', item[DATASTORE_COLUMN_PATIENT_ID])
Beispiel #29
0
def api_home():
    request_info = extract_request(request, session)
    if request.method == 'POST':
        message = request.form.get('message', '')
        datastore = request.form.get('datastore', '')
        DataStore.store_message(datastore, message)
    context = {'request_info': request_info}
    response = make_response(json.dumps(context))
    response.set_cookie('dt', get_now())
    return response
Beispiel #30
0
    def __init__(self,
                 entity_name=None,
                 source_language_script=lang_constant.ENGLISH_LANG,
                 translation_enabled=False):
        """
        Initializes a TextDetector object with given entity_name

        Args:
            entity_name: A string by which the detected substrings that correspond to text entities would be replaced
                         with on calling detect_entity()
            source_language_script: ISO 639 code for language of entities to be detected by the instance of this class
            translation_enabled: True if messages needs to be translated in case detector does not support a
                                 particular language, else False
        """
        # assigning values to superclass attributes
        self._supported_languages = [
            lang_constant.ENGLISH_LANG,
            lang_constant.HINDI_LANG,
            lang_constant.GUJARATI_LANG,
            lang_constant.HINDI_LANG,
            lang_constant.ENGLISH_LANG,
            lang_constant.GUJARATI_LANG,
            lang_constant.BENGALI_LANG,
            lang_constant.MARATHI_LANG,
            lang_constant.TELEGU_LANG,
            lang_constant.TAMIL_LANG,
            lang_constant.URDU_LANG,
            lang_constant.KANNADA_LANG,
            lang_constant.ORIYA_LANG,
            lang_constant.MALAYALAM_LANG,
            lang_constant.
            PUNJABI_LANG  # Added temporarily till text detection is ported to v2 api
        ]
        super(TextDetector, self).__init__(source_language_script,
                                           translation_enabled)
        self.tagged_text = None
        self.processed_text = None
        self.__texts = []
        self.__tagged_texts = []
        self.__processed_texts = []

        self.entity_name = entity_name
        self.tag = '__' + self.entity_name + '__'

        # defaults for auto mode
        self._fuzziness = "auto:4,7"
        self._fuzziness_lo, self._fuzziness_hi = 4, 7
        self._min_token_size_for_fuzziness = self._fuzziness_lo
        # self.set_fuzziness_threshold(fuzziness=(self._fuzziness_lo, self._fuzziness_hi))

        # defaults for non-auto mode
        self.set_fuzziness_threshold(fuzziness=1)
        self._min_token_size_for_fuzziness = 4

        self.db = DataStore()
Beispiel #31
0
def preprocess(dbPath):
    '''
	This is a preprocess module
	'''
    logging = DefaultLogger()

    if not os.path.exists(dbPath):
        logging.debug('PreProcess: can\'t find database at path')
        return

    datastore = DataStore(dbPath)
    loopcount = 0

    while True:
        sleep(5)

        if loopcount % 10 == 0:
            logging.debug('PreProcess is alive')
        loopcount += 1

        data = datastore.recordsForHashing()
        for record in data:
            logging.debug(record)

            key_id = record.id
            filePath = record.fileName

            if not os.path.exists(filePath):
                logging.debug(
                    'PreProcess: Will update record status as the file no longer exists'
                )
                datastore.updateRecordAsMissingWithID(key_id)
                continue

            try:
                logging.debug('PreProcess: locking file to calculate hash...')
                ##UPDATE HASH OPERATION START HERE
                startTime = datetime.datetime.now().strftime(
                    "%Y-%m-%d %H:%M:%S")
                datastore.updateRecordWithHashStart(startTime, key_id)

                fileToHash = open(filePath, 'rb')
                portalocker.lock(fileToHash, portalocker.LOCK_SH)
                hashString = "NO_OP"  #hashForFile(fileToHash)
                endTime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                fileToHash.close()

                logging.debug('PreProcess: unlocking file...')
                logging.debug(
                    'PreProcess: Will update record status with Hash string and times'
                )

                datastore.updateRecordWithHashForStartTimeAndEndTime(
                    hashString, startTime, endTime, key_id)

            except Exception as e:
                info = 'PreProcess: There was an error when calculating the hash for file: ' + os.path.basename(
                    filePath) + ' ' + e.message
                sendFailureEmail(info)
                logging.error(e.message)
class Session:
    def __init__(self):
        """Constructor for the session class to store information regarding the session table
        
        Args:
            None

        Returns:
            The Session object
        """
        self._databaseName = "dynamodb"
        self._tableName = "Sessions"
        self._partitionKeyName = "PatientId"
        self._sortKeyName = "SessionId"
        self._indexName = "hcpIndex"
        self._indexPartitionKeyName = "HealthCareProfessionalId"
        self._indexSortKeyName = "SessionId"
        self._sessionDataStore = DataStore(self._databaseName, self._tableName, self._partitionKeyName, self._sortKeyName, self._indexName, self._indexPartitionKeyName, self._indexSortKeyName)
    
    def createSession(self, info):
        """Create the session and store the data into database
        
        Args:
            info(dict): information regarding the session (patient id, session id, health care professional id, session name etc.)

        Returns:
            None
        """
        return self._sessionDataStore.save(info)

    def requestSession(self, partitionKey, sortKey, indexPartitionKey):
        """Request and get the specific session(s) from the database.
        If both partition key and sort key are provided, return that single specific session.
        if both index partition key and sort key are provided, return that single specific session.
        if only partition key is provided, return sessions with that partition key.
        if onyl index partition key is provided, return sessions with that index partition key.
        
        Args:
            partitionKey(str): partition key value of the session table
            sortKey(str): sort key value of the session table
            indexPartitionKey(str): index partition key value of the session table

        Returns:
            List of session(s) info
        """
        if partitionKey and sortKey:
            return self._sessionDataStore.queryByBothKeys(partitionKey, sortKey)
        elif indexPartitionKey and sortKey:
            return self._sessionDataStore.queryByIndexBothKeys(indexPartitionKey, sortKey)
        elif partitionKey:
            return self._sessionDataStore.queryByPartitionKey(partitionKey)
        elif indexPartitionKey:
            return self._sessionDataStore.queryByIndexPartitionKey(indexPartitionKey)

        return {"Items": [], "status": "BAD", "message": "Wrong Way to Search."}
Beispiel #33
0
def update_poi_test(id):
    store = DataStore()
    poi = PointOfInterest.get_test_item(id)
    poi.name = "t_name_toUpdate"
    poi.category = "t_category_toUpdate"
    poi.creator = "t_creator_toUpdate"
    poi.description = "_description_toUpdate"
    poi.latitude = 20.00000
    poi.longitude = 10.00000
    poi = store.update_poi(poi)
    return to_json(poi)
 def test_open_data_store_raises_FileNotFound(self):
     with self.assertRaisesRegex(
             FileNotFoundError,
             "The root of the data store: 'testing/data/does_not_exist' does not exist"
     ):
         ds = DataStore('testing/data/does_not_exist')
     with self.assertRaisesRegex(
             FileNotFoundError,
             "The root of the data store: 'testing/data/xxx' is not a directory"
     ):
         ds = DataStore('testing/data/xxx')
 def setUp(self):
     self.datastore = DataStore(initial_dataset=[
         {
             'id': 1,
             'name': 'Nemeziz',
         },
         {
             'id': 2,
             'name': 'Copa Mundial',
         },
     ], )
	def initialize_connections(self):
		if len(self.socket_table) < TaskHandler.PEER_REPICK:
			if DBGMODE : print("initialize_connections [start]")
			DataStore.mutex.acquire()
			self.datastore = DataStore()
			self.peers = self.datastore.get_highest_rating(10)
			for peer in self.peers:
				if peer.id not in self.socket_table and (peer.ip != self.remote_ip or peer.port !=self.remote_port):
					self.socket_table[peer.id] = (peer.ip, int(peer.port))
			self.datastore.close()
			DataStore.mutex.release()
Beispiel #37
0
    def setUp(self):
        db = sqlite3.connect(":memory:")

        # Populate the datastore
        self.ds = DataStore(db)
        data = [1, 2, 3, 4, 5]
        self.ds.segments.addSegment(0x1234, len(data), "ROM", data)
        self.seg = self.ds.segments.__iter__().next()
        del self.ds

        # Reload the datastore from the db
        self.ds = DataStore(db)
Beispiel #38
0
 def test_save_collision(self):
     table = self.create_session_table()
     d = DataStore('dynamodb', 'Sessions', 'PatientId', 'SessionId')
     d.save(SESSION_INFO_1)
     response = table.scan()
     self.assertEqual(1, len(response['Items']))
     self.assertEqual('SessionName', response['Items'][0]['SessionName'])
     response = d.save(SESSION_INFO_1_SAMEKEY)
     self.assertEqual('OK', response['status'])
     response = table.scan()
     self.assertEqual(1, len(response['Items']))
     self.assertEqual('NO', response['Items'][0]['SessionName'])
 def __init__(self):
     try:
         self.auth = OAuthHandler(Secretes._twitterConsumerApiKey,
                                  Secretes._twitterConsumerApiSecret)
         self.auth.secure = True
         self.auth.set_access_token(Secretes._twitterAccessToken,
                                    Secretes._twitterAccessTokenSecret)
         self.api = API(self.auth)
         self.dataStore = DataStore()
     except BaseException as e:
         print("Error in main()", e)
     self.tweetCount = 0
 def __init__(self):
     try:
         logging.basicConfig(filename=GlobalVariables._logFileName)
         self.auth = OAuthHandler(Secretes._twitterConsumerApiKey,
                                  Secretes._twitterConsumerApiSecret)
         self.auth.secure = True
         self.auth.set_access_token(Secretes._twitterAccessToken,
                                    Secretes._twitterAccessTokenSecret)
         self.api = API(self.auth)
         self.dataStore = DataStore()
     except BaseException as e:
         print("Error in main()", e)
     self.tweetCount = 0
Beispiel #41
0
 def test3(self):
     con = make_dbcon()
     ds = DataStore(con)
     
     users = ds.collection("users")  
     users.add_index("email")
     
     uuid = users.save({'email': '*****@*****.**', 'name':'John Doe!'})
     self.assertIsNotNone(users.index("email").find("*****@*****.**"))
     
     users.delete(uuid)
     
     self.assertEqual(len(users.index("email").find("*****@*****.**")), 0)
Beispiel #42
0
def start():
    fileConfig('conf/log.conf')
    logging.getLogger('garage').log(logging.DEBUG, 'Log setup complete')

    logging.getLogger('garage').log(logging.DEBUG, 'Initializing datastore ')
    db = DataStore(setup='true')
    db.shutdown()
    logging.getLogger('garage').log(logging.DEBUG, 'Complete')

    #butler = None
    butler = Butler()

    web.start(butler)
def get_tasks():
	u = request.form['url'].lower()
	
	url = Utilities.get_shortened_url(u)
	url_3 = Utilities.get_shortened_url(u,3)

	return_only_parent = False

	# If url is same as parent url, return everything just for parent
	# Dont redundantly return for parent and itself
	if url == url_3 or url+'/' == url_3:
			return_only_parent = True

	ds = DataStore()

	if not return_only_parent:

		all_urls = Utilities.modify_url(url)
		print all_urls

		# If the same url is also a parent url, return all results of parent .
		# And skip individual url results

		for url in all_urls:
			result = ds.fetch(url)
			if result == False:
				print " Tried for url " + url
			else:
				x = {"result":result}
				return jsonify(x)

	# If for our exact url and its modifications , nothing got returned

	outer_url = "parent::" + Utilities.get_shortened_url(url,3)
	print outer_url
	
	result = ds.fetch_all_from_parent(outer_url)
	if result : 
		x = {"result":result}
		return jsonify(x)
	else:
		if outer_url[-1] == '/':
			result = ds.fetch_all_from_parent(outer_url[:-1])
		else:
			result = ds.fetch_all_from_parent(outer_url + '/')
		if result : 
			x = {"result":result}
			return jsonify(x)

	# If there is still nothing to show
	return 'No Response'
Beispiel #44
0
    def test2(self):
        con = make_dbcon()
        ds = DataStore(con)
        
        col = ds.collection("users")  
        col.add_index("email")
        
        uuid = col.save({'email': '*****@*****.**', 'name':'My Name!'})

        i = col.index('email')
        self.assertIsNone(i.findone('*****@*****.**'))
        
        doc = i.findone('*****@*****.**')
        self.assertEqual(doc['uuid'], uuid)
Beispiel #45
0
def get_pois():
    store = DataStore()
    query_dic = dict()
    use_query = False
    for k in PointOfInterest().__dict__.keys():
        v = request.args.get(k, None)
        if v is not None:
            query_dic[k] = v
            use_query = True
    if use_query:
        pois = store.get_poi_filtered(query_dic)
    else:
        pois = store.get_all_poi()
    return to_json(pois)
def handle_put(seq,key, value):
    """Return a tuple containing True and the message
    to send back to the client."""

    if key not in POROCESSING:
        POROCESSING.append(key)
        ds = DataStore(key,value)
        if ds.put(seq):
            POROCESSING.remove(key)
            return (True, 'Key [{}] set to [{}]'.format(key, value))
        else:
            ds.roll_back(seq)
            POROCESSING.remove(key)
    return (False, 'Could Not be added')
def handle_delete(seq,key):
    """Return a tuple containing True if the key could be deleted and
    the message to send back to the client.

    use datastore.get and then before doing doing datastore.delete
    """
    ds = DataStore()
    if ds.get(key):
        POROCESSING.append(key)
        if ds.delete(seq,key):   
            POROCESSING.remove(key)
            return (True,'Done')
        else:
            ds.roll_back(seq) 
            POROCESSING.remove(key)
    return (False,'ERROR: Key [{}] not found and could not be deleted'.format(key))
Beispiel #48
0
 def test4(self):
     con = make_dbcon()
     ds = DataStore(con)
     
     users = ds.collection("users")  
     users.add_index("email")
     
     uuid = users.save({'email': '*****@*****.**', 'name':'John Doe!'})
     doc = users.load(uuid)
     
     self.assertIsNotNone(users.index.email.findone('*****@*****.**'))
     
     del doc['email']
     users.save(doc)
     
     self.assertIsNone(users.index.email.findone('*****@*****.**'))
Beispiel #49
0
 def test2(self):
     con = make_dbcon()
     ds = DataStore(con)
     
     col = ds.collection("users")
     uuid = col.save({
     'email':'*****@*****.**', 
     'name':'Roel Gerrits'
     })
             
     
     doc = col.load(uuid)
     
     self.assertDictEqual(doc, {
     'uuid': uuid, 
     'email':'*****@*****.**', 
     'name':'Roel Gerrits'
     })
Beispiel #50
0
    def test3(self):
        con = make_dbcon()
        ds = DataStore(con)
        
        col = ds.collection("users")
        uuid = col.save({
        'email':'*****@*****.**', 
        'name':'Roel Gerrits'
        })
                
        
        self.assertTrue(col.delete(uuid))
        self.assertFalse(col.delete(uuid))

        
        doc = col.load(uuid)
        
        self.assertIsNone(doc)
Beispiel #51
0
def preprocess(dbPath):
	'''
	This is a preprocess module
	'''
	logging = DefaultLogger()

	if not os.path.exists(dbPath):
		logging.debug('PreProcess: can\'t find database at path')
		return

	datastore = DataStore(dbPath)
	loopcount = 0

	while True:
		sleep(5)

		if loopcount % 10 == 0:
			logging.debug('PreProcess is alive')
		loopcount += 1

		data = datastore.recordsForHashing()
		for record in data:
			logging.debug(record)

			key_id = record.id
			filePath = record.fileName

			if not os.path.exists(filePath):
				logging.debug('PreProcess: Will update record status as the file no longer exists')
				datastore.updateRecordAsMissingWithID(key_id)
				continue

			try:
				logging.debug('PreProcess: locking file to calculate hash...')
				##UPDATE HASH OPERATION START HERE
				startTime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
				datastore.updateRecordWithHashStart(startTime, key_id)

				fileToHash = open(filePath, 'rb')
				portalocker.lock(fileToHash, portalocker.LOCK_SH)
				hashString = "NO_OP"#hashForFile(fileToHash) 
				endTime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
				fileToHash.close()

				logging.debug('PreProcess: unlocking file...')
				logging.debug('PreProcess: Will update record status with Hash string and times')

				datastore.updateRecordWithHashForStartTimeAndEndTime(hashString, startTime, endTime, key_id)

			except Exception as e:
				info = 'PreProcess: There was an error when calculating the hash for file: ' + os.path.basename(filePath) + ' ' + e.message
				sendFailureEmail(info)
				logging.error(e.message)
Beispiel #52
0
    def test1(self):
        con = make_dbcon()
        ds = DataStore(con)

        col = ds.collection("users")
        col.save({
        'uuid': 'f8e5519ee55811e2839a0024219c6b57', 
        'email':'*****@*****.**', 
        'name':'Roel Gerrits'
        })
                
        
        doc = col.load('f8e5519ee55811e2839a0024219c6b57')
        
        self.assertDictEqual(doc, {
        'uuid': 'f8e5519ee55811e2839a0024219c6b57', 
        'email':'*****@*****.**', 
        'name':'Roel Gerrits'
        })
Beispiel #53
0
 def __init__(self, filename):
     """Initializes a Text."""
     assert filename
     self.filename = filename
     self.text = self.get_text_from_txt_file()
     
     if DataStore.is_to_be_computed(filename):
         database = DataStore(filename, self.__compute_ds_keys_values())
     
     else:
         database = DataStore(filename)
     
     self.ds = database
Beispiel #54
0
	def update_database(self, peers):
		DataStore.mutex.acquire()
		database_conn = DataStore()
		current_nodes = database_conn.get_all_peers()
		new_list = []
		for peer in peers:
			overlay = False
			for node in current_nodes:
				if node.ip == peer["ip"] and node.port == peer["port"]:
					if node.rating < 30:
						node.rating = 30
					database_conn.update(node)
					overlay = True

			if overlay == False:
				new_list.append(PeerNode(0, peer["id"], peer["ip"], peer["port"], 30))
		database_conn.insert_peers(new_list)
		database_conn.close()
		DataStore.mutex.release()
    def __init__(self):
        super(CalendarPlugin, self).__init__()
        self.first_day = self.last_day = self.numdays = None

        builder = Gtk.Builder()
        builder.add_from_file("calendar_view.glade")
        handlers = {
            "on_window_destroy": Gtk.main_quit,
            "on_today_clicked": self.on_today_clicked,
            "on_combobox_changed": self.on_combobox_changed,
            "on_add_clicked": self.on_add_clicked,
            "on_edit_clicked": self.on_edit_clicked,
            "on_remove_clicked": self.on_remove_clicked,
            "on_next_clicked": self.on_next_clicked,
            "on_previous_clicked": self.on_previous_clicked,
        }
        builder.connect_signals(handlers)

        self.window = builder.get_object("window")
        self.window.__init__()
        self.window.set_title("GTG - Calendar View")
        self.window.connect("destroy", Gtk.main_quit)

        # DataStore object
        self.ds = DataStore()
        self.req = Requester(self.ds)
        self.ds.populate()  # hard-coded tasks

        self.today_button = builder.get_object("today")
        self.header = builder.get_object("header")

        self.controller = Controller(self, self.req)
        vbox = builder.get_object("vbox")
        vbox.add(self.controller)
        vbox.reorder_child(self.controller, 1)

        self.current_view = None
        self.combobox = builder.get_object("combobox")
        self.combobox.set_active(2)

        self.statusbar = builder.get_object("statusbar")

        self.window.show_all()
Beispiel #56
0
            "reference:": reference,
            "title": documents[0]["title"].decode("utf-8"),
            "date": datestring,
            "attachment": solr_attachments.values(),
            "session": solr_sessions.values(),
            "committee": solr_committees.values(),
            "street": unique(solr_streets.keys()),
            "person": solr_people.values(),
            "content": solr_body,
            #'location': positions_for_streetnames(unique(solr_streets.keys()))
        }


if __name__ == "__main__":
    s = solr.SolrConnection(config.SOLR_URL)
    db = DataStore(config.DBNAME, config.DBHOST, config.DBUSER, config.DBPASS)
    parser = OptionParser()
    parser.add_option(
        "-s",
        "--sample",
        dest="sample",
        default=1,
        help="z.B. die Zahl 10 um nur jedes zehnte Dokument zu importieren. Beschleunigt den Import beim Entwickeln.",
    )
    parser.add_option(
        "-v",
        "--verbose",
        dest="verbose",
        default=False,
        action="store_true",
        help="Aktiviert die detailliertere Ausgabe (Dokumententitel etc.).",
	def on_data(self,data):
		try:

			data = json.loads(data)
			newd = {}

			# Get Tweet
			tweet = Utilities.clean_tweet(data['text'])

			for key in self.recent_tweets:
				#print Utilities.similarity(key,tweet)
				if Utilities.similarity(key,tweet) > 70:
					return
			'''
			if tweet in self.recent_tweets:
				return
			else:
			'''
			if len(self.recent_tweets) > 50:
				self.recent_tweets.popitem(last=False)
			self.recent_tweets[tweet] = True
			#print tweet

			# Get Redirected url
			try:
				url_name = Utilities.get_redirected_url(str(data['entities']['urls'][0]['expanded_url']))
			except:
				return
				raise BaseException("Url for tweet did not exist")

			# Get shortened url for key --> Upto 5th '/' or entire address (whichever is shorter)

			url_name = Utilities.get_shortened_url(url_name).lower()

			#Get timestamp
			timestamp = str(data['created_at'])

			# Verify authenticity of website by checking if it has the word coupon
			# If it does , assume it is not a vendor site. Maybe blog, maybe coupon site

			try:
				Utilities.check_url_validity(url_name)
			except:
				return
				raise BaseException("Url was not a valid site")


			with open("x.txt","a") as f:
				f.write(tweet + '\n')
				f.write("--------------------" + '\n')
			# Code to extract important information from this tweet
			#self.tweets += 1
			#print tweet
			#print "Tweet Number : " + str(self.tweets)
			e = Extraction()
			code,date = e.extract_all(tweet)
			if not code:
				#print " --------------- "
				return
				raise BaseException("Did not have coupon code information")

			if not date :
				date = 183600
			else :
				self.tweets_with_dates += 1
				self.total_expiry_time += date
				self.exp_time.append(date/3600)
				print self.tweets_with_dates, int(numpy.median(self.exp_time))
				#print date
				#print self.tweets_with_dates
				print tweet
				#print " ----------------------------------- "
				#print "Tweet : ",

				#print "Url : ",
				#print url_name
				#print "Date : "


			#print "Coupons : " + str(self.tweets_with_coupons)
			#print "Dates : " + str(self.tweets_with_dates)
			#print "Total Expiry Time :" + str(self.total_expiry_time/3600) + "hours"
			#print "Avg Expiry Time :" + str((self.total_expiry_time/(self.tweets_with_dates+1))/3600) + "hours"
			print '--------------------------------------'

			#print "CODE : " + code
			key = url_name + ':::' + code
			#print "KEY : " + key

			#print "Tweet : "
			#print tweet
			#print "Url : ",
			#print url_name
			#print " ----------------------------------- "

			ds = DataStore()
			#print url_name,code,date
			#get outer url - url uptil 3 '/'s . eg - http://www.etsy.com/
			outer_url = "parent::"+Utilities.get_shortened_url(url_name,3)
			ds.insert(key,url_name,code,tweet,date,outer_url)
			#print '-----------------------'

			return True
		except BaseException as e:
			if str(e) != "'text'":
				#print " *************** " + str(e) + " *************** "
				#print "----------------------------------------"
				pass
			time.sleep(1)
def handle_keys():
    ds = DataStore()
    keys = ds.mykeys()
    keys = [x[0] for x in keys]
    keys_n = '-'.join(str(x) for x in keys) 
    return (True,keys_n)