예제 #1
0
 def token_cb(token):
     self.indx_con = IndxClient(self.config['address'],
                                self.config['box'],
                                appid,
                                token=token,
                                client=authclient.client)
     return_d.callback(True)
예제 #2
0
파일: client.py 프로젝트: indx/indx-core
 def token_cb(token):
     if not self.indx:
         self.indx = IndxClient(self.args['server'],
                                self.args['box'],
                                self.appid,
                                token=token,
                                client=self.client)
     do_call()
예제 #3
0
    def call_action(self, name, *args, **kwargs):
        """ Calls an action by name. """

        action = self.actions[name]
        f = action['f']
        self.check_args(action['args'])

        if not self.indx:
            self.indx = IndxClient(self.args['server'], self.args['box'],
                                   self.args['username'],
                                   self.args['password'], self.appid)

        return self.parse_status(name, f(*args, **kwargs))
예제 #4
0
파일: client.py 프로젝트: indx/indx-core
    def call_action(self, name, *args, **kwargs):
        """ Calls an action by name. """
        return_d = Deferred()

        action = self.actions[name]
        f = action['f']
        self.check_args(action['args'])

        def do_call():
            f(*args, **kwargs).addCallbacks(
                lambda status: return_d.callback(
                    self.parse_status(name, status)), return_d.errback)

        if not self.token:

            def token_cb(token):
                if not self.indx:
                    self.indx = IndxClient(self.args['server'],
                                           self.args['box'],
                                           self.appid,
                                           token=token,
                                           client=self.client)
                do_call()

            self.auth_and_get_token(IndxClient.requires_token(f)).addCallbacks(
                token_cb, return_d.errback)
        else:
            do_call()

        return return_d
예제 #5
0
 def token_cb(token):
     indx = IndxClient(server_url,
                       self.config_box,
                       "INDX_Fitbit_Harvester",
                       token=token,
                       client=authclient.client)
     indx_d.callback(indx)
예제 #6
0
파일: import.py 프로젝트: kyonetca/indx
 def token_cb(token):
     indx = IndxClient(server_url,
                       box,
                       "Slicer",
                       token=token,
                       client=authclient.client)
     logging.debug("Got the indx client token")
     indx_d.callback(indx)
예제 #7
0
파일: client.py 프로젝트: raminetinati/indx
    def call_action(self, name, *args, **kwargs):
        """ Calls an action by name. """

        action = self.actions[name]
        f = action['f']
        self.check_args(action['args'])

        if not self.indx:
            self.indx = IndxClient(self.args['server'], self.args['box'], self.args['username'], self.args['password'], self.appid)

        return self.parse_status(name, f(*args, **kwargs))
예제 #8
0
파일: sync.py 프로젝트: kyonetca/indx
                                def token_cb(remote_token):
                                    logging.debug("IndxSync sync_boxes token_cb")
                                    client = IndxClient(remote_server_url, remote_box, self.APPID, client = clientauth.client, token = remote_token, keystore = self.keystore)
                                    
                                    def updated_cb(empty):

                                        def pushed_cb(empty):

                                            def observer(data):

                                                if data.get('action') == 'diff' and data.get('operation') == 'update':
                                                    diff = data['data']

                                                    def done_cb(empty):
                                                        logging.debug("IndxSync updating from a websocket done.")
                                                    def err_cb(failure):
                                                        logging.error("IndxSync updating from a websocket error: {0}".format(failure))

                                                    self.update_to_latest_version(client, remote_server_url, remote_box, diff_in = diff).addCallbacks(done_cb, err_cb)                                  
                                                else:
                                                    logging.error("Sync: Unknown data message from WebSocket: {0}".format(data))

                                            # auths and sets up listening for diffs, filtering them and passing them to the observer
                                            if websocket is None:
                                                wsclient = client.connect_ws(local_key['key']['private'], local_key_hash, observer, remote_encpk2, self.webserver) # open a new socket
                                            else:
                                                websocket.listen_remote(local_key['key']['private'], local_key_hash, observer, remote_encpk2) # use an existing websocket

                                            next_model(None)

                                        if not include_push_all:
                                            pushed_cb(None)
                                            return
                                        
                                        # push the whole box to the remote box
                                        def latest_cb(graph):
                                            def ver_cb(version):
                                                client.update_raw(version, graph.to_flat_json()).addCallbacks(pushed_cb, return_d.errback)

                                            client.get_version().addCallbacks(lambda resp: ver_cb(resp['data']), return_d.errback)

                                        self.root_store.get_latest(render_json = False).addCallbacks(latest_cb, return_d.errback)

                                    # compare local version to previous, and update one of them, or both
                                    self.update_to_latest_version(client, remote_server_url, remote_box).addCallbacks(updated_cb, return_d.errback)
예제 #9
0
파일: client.py 프로젝트: allmende/indx
    def call_action(self, name, *args, **kwargs):
        """ Calls an action by name. """
        return_d = Deferred()

        action = self.actions[name]
        f = action['f']
        self.check_args(action['args'])

        def do_call():
            f(*args, **kwargs).addCallbacks(lambda status: return_d.callback(self.parse_status(name, status)), return_d.errback)

        if not self.token:
            def token_cb(token):
                if not self.indx:
                    self.indx = IndxClient(self.args['server'], self.args['box'], self.appid, token = token, client = self.client)
                do_call()

            self.auth_and_get_token(IndxClient.requires_token(f)).addCallbacks(token_cb, return_d.errback)
        else:
            do_call()
            
        return return_d
예제 #10
0
파일: sync.py 프로젝트: kyonetca/indx
        def ver_cb(ver):

            def new_remote_key_cb(remote_keys):
                logging.debug("IndxSync link_remote_box, new_remote_key_cb, remote_keys: {0}".format(remote_keys))
                # NB: no "private" in remote_keys, that never leaves the remote box.
                remote_keys = remote_keys['data'] # remove the Indx HTTP response padding

                remote_encpk = remote_keys['encpk2']
                if type(remote_encpk) != type(""):
                    remote_encpk = json.dumps(remote_encpk) 

                remote_serverid = remote_keys['serverid']

                link_uid = uuid.uuid1()
                local_key_uid = uuid.uuid1()
                remote_key_uid = uuid.uuid1()
                status1_uid = uuid.uuid1()
                status2_uid = uuid.uuid1()

                link_uri = "link-{0}".format(link_uid)
                status1_uri = "status-{0}".format(status1_uid)
                status2_uri = "status-{0}".format(status2_uid)

                # objects get updated to local box, and then synced across to the other boxes once the syncing starts
                new_objs = [
                    {   "@id": link_uri,
                        "type": [ {"@value": NS_ROOT_BOX + "link"} ],
                        "boxes": [ {"@id": "box-{0}".format(local_key_uid)}, # links to the objs below
                                   {"@id": "box-{0}".format(remote_key_uid)}, # links to the objs below
                                 ],
                        "statuses": [ {"@id": status1_uri}, {"@id": status2_uri} ],
                    },
                    { "@id": status1_uri,
                      "type": [ {"@value": NS_ROOT_BOX + "status"} ],
                      "src-boxid": [ {"@value": remote_box} ],
                      "src-server-url": [ {"@value": remote_address} ],
                      "dst-boxid": [ {"@value": self.root_store.boxid} ],
                      "dst-server-url": [ {"@value": self.url} ],
                      "last-version-seen": [ {"@value": ver + 1} ], #we save this because we will push our whole box to it in a sec - it is incremented because the version is before we put this in
                    },
                    { "@id": status2_uri,
                      "type": [ {"@value": NS_ROOT_BOX + "status"} ],
                      "dst-boxid": [ {"@value": remote_box} ],
                      "dst-server-url": [ {"@value": remote_address} ],
                      "src-boxid": [ {"@value": self.root_store.boxid} ],
                      "src-server-url": [ {"@value": self.url} ],
                    },
                    {   "@id": "box-{0}".format(local_key_uid),
                        "type": [ {"@value": NS_ROOT_BOX + "box"} ],
                        "server-url": [ {"@value": self.url } ],
                        "server-id": [ {"@value": server_id } ],
                        "box": [ {"@value": self.root_store.boxid} ],
                        "key": [ {"@id": local_keys['public-hash']}], # links to the key objs below
                    },
                    {   "@id": "box-{0}".format(remote_key_uid),
                        "type": [ {"@value": NS_ROOT_BOX + "box"} ],
                        "server-url": [ {"@value": remote_address } ],
                        "server-id": [ {"@value": remote_serverid } ],
                        "box": [ {"@value": remote_box} ],
                        "key": [ {"@id": remote_keys['public-hash']}], # links to the key objs below
                    },
                    {   "@id": local_keys['public-hash'],
                        "type": [ {"@value": NS_ROOT_BOX + "key"} ],
                        "public-key": [ {"@value": local_keys['public']} ], # share the full public keys everywhere (private keys only in respective server's keystores)
                        "public-hash": [ {"@value": local_keys['public-hash']} ], # share the full public keys everywhere (private keys only in respective server's keystores)
                    },
                    {   "@id": remote_keys['public-hash'],
                        "type": [ {"@value": NS_ROOT_BOX + "key"} ],
                        "public-key": [ {"@value": remote_keys['public']} ], # share the full public keys everywhere
                        "public-hash": [ {"@value": remote_keys['public-hash']} ], # share the full public keys everywhere
                    },
                ]



                def update_cb(empty):

                    def diff_cb(diff):

                        def applied_cb(empty):

                            def encpk_cb(empty):

                                def remote_added_cb(empty):

                                    def local_added_cb(empty):
                                        logging.debug("IndxSync link_remote_box, local_added_cb")

                                        def added_indx_cb(empty):
                                            # start syncing/connecting using the new key
                                            self.sync_boxes([link_uri], include_push_all = True).addCallbacks(lambda empty: return_d.callback(remote_keys['public']), return_d.errback)

                                        self.database.save_linked_box(self.root_store.boxid).addCallbacks(added_indx_cb, return_d.errback)

                                    # add the local key to the local store
                                    self.keystore.put(local_keys, local_user, self.root_store.boxid).addCallbacks(local_added_cb, return_d.errback) # store in the local keystore

                                self.keystore.put({"public": remote_keys['public'], "private": "", "public-hash": remote_keys['public-hash']}, local_user, self.root_store.boxid).addCallbacks(remote_added_cb, return_d.errback)

                            # don't save the local encpk2 here, only give it to the remote server.
                            # save the remote encpk2
                            self.database.save_encpk2(sha512_hash(remote_encpk), remote_encpk, remote_serverid).addCallbacks(encpk_cb, return_d.errback)

                        client.apply_diff(diff).addCallbacks(applied_cb, return_d.errback)

                    self.root_store.diff(0, None, "diff").addCallbacks(diff_cb, return_d.errback)

                self.root_store.update(new_objs, ver, propagate = True).addCallbacks(update_cb, return_d.errback)

            client = IndxClient(remote_address, remote_box, self.APPID, token = remote_token)
            client.generate_new_key(local_keys, local_encpk, server_id).addCallbacks(new_remote_key_cb, return_d.errback)
예제 #11
0
파일: client.py 프로젝트: indx/indx-core
class CLIClient:
    def __init__(self, appid="INDX CLIClient"):
        """ Associate command-line actions with functions, and enforce required variables. """
        self.actions = {
            'create_box': {
                'f': self.create_box,
                'args': ['box']
            },
            'delete_box': {
                'f': self.delete_box,
                'args': ['box']
            },
            'list_boxes': {
                'f': self.list_boxes,
                'args': []
            },
            'get_object_ids': {
                'f': self.get_object_ids,
                'args': ['box']
            },
            'create_user': {
                'f': self.create_user,
                'args': ['target_username', 'target_password']
            },
            'update': {
                'f': self.update,
                'args': ['box', 'data', 'version']
            },
            'update_json': {
                'f': self.update_json,
                'args': ['box', 'data', 'version']
            },
            'update_raw': {
                'f': self.update_raw,
                'args': ['box', 'data', 'version']
            },
            'delete': {
                'f': self.delete,
                'args': ['box', 'id', 'version']
            },
            'get_latest': {
                'f': self.get_latest,
                'args': ['box']
            },
            'get_by_ids': {
                'f': self.get_by_ids,
                'args': ['box', 'id']
            },
            'query': {
                'f': self.query,
                'args': ['box', 'query']
            },
            'diff': {
                'f': self.diff,
                'args': ['box', 'from', 'return_objs']
            },
            #'listen': {'f': self.listen, 'args': ['box']},
            'add_file': {
                'f': self.add_file,
                'args': ['box', 'data', 'id', 'version', 'contenttype']
            },
            'delete_file': {
                'f': self.delete_file,
                'args': ['box', 'id', 'version']
            },
            'get_file': {
                'f': self.get_file,
                'args': ['box', 'id']
            },
            'list_files': {
                'f': self.list_files,
                'args': ['box']
            },
            'set_acl': {
                'f': self.set_acl,
                'args': ['box', 'acl', 'target_username']
            },
            'set_acl_public': {
                'f': self.set_acl_public,
                'args': ['box', 'acl']
            },
            'get_acls': {
                'f': self.get_acls,
                'args': ['box']
            },
            'generate_new_key': {
                'f': self.generate_new_key,
                'args': ['box']
            },
            'create_root_box': {
                'f': self.create_root_box,
                'args': ['box']
            },
            'link_remote_box': {
                'f': self.link_remote_box,
                'args':
                ['box', 'remote_token', 'remote_box', 'remote_address']
            },
            'generate_token': {
                'f': self.generate_token,
                'args': ['box']
            },
        }

        self.token = None
        self.appid = appid
        self.indx = None

    def set_args(self, args):
        """ Move relevant command-line arguments into local variable. """
        logging.debug("Set args: {0}".format(args))
        self.args = args
        """ Flatten single value lists into flat key/value pair. """
        for key in args:
            if type(args[key]) == type([]) and len(args[key]) == 1:
                args[key] = args[key][0]
        """ Ensure self.server always ends in a / """
        if self.args['server'][-1:] != "/":
            self.args['server'] += "/"

    def check_args(self, required):
        not_set = []
        for key in required:
            if key not in self.args or self.args[key] is None or self.args[
                    key] == "":
                not_set.append(key)

        if not self.args['allowempty'] and len(not_set) > 0:
            raise Exception(
                "The following values cannot be empty for this action: {0}".
                format(", ".join(not_set)))

    def auth_and_get_token(self, get_token):
        """ Authenticate, get a token and call it back to the deferred. """
        return_d = Deferred()

        def authed_cb():
            def token_cb(token):
                if token is not None:
                    self.token = token
                    return_d.callback(token)
                else:
                    return_d.callback(None)

            if get_token:
                authclient.get_token(self.args['box']).addCallbacks(
                    token_cb, return_d.errback)
            else:
                token_cb(None)

        authclient = IndxClientAuth(self.args['server'], self.appid)
        self.client = authclient.client
        authclient.auth_plain(self.args['username'],
                              self.args['password']).addCallbacks(
                                  lambda response: authed_cb(),
                                  return_d.errback)

        return return_d

    def call_action(self, name, *args, **kwargs):
        """ Calls an action by name. """
        return_d = Deferred()

        action = self.actions[name]
        f = action['f']
        self.check_args(action['args'])

        def do_call():
            f(*args, **kwargs).addCallbacks(
                lambda status: return_d.callback(
                    self.parse_status(name, status)), return_d.errback)

        if not self.token:

            def token_cb(token):
                if not self.indx:
                    self.indx = IndxClient(self.args['server'],
                                           self.args['box'],
                                           self.appid,
                                           token=token,
                                           client=self.client)
                do_call()

            self.auth_and_get_token(IndxClient.requires_token(f)).addCallbacks(
                token_cb, return_d.errback)
        else:
            do_call()

        return return_d

    def parse_status(self, source, status):
        """ Parse the status returned from the server, and raise an Exception if necessary. """

        if status is not None:  # status is None when a file has been printed raw
            if status['code'] < 200 or status['code'] > 299:
                raise Exception(
                    "{0} in box {1} failed. Response is {2} with code {3}".
                    format(source, self.args['box'], status['message'],
                           status['code']))
            else:
                if "data" in status and self.args['jsondata']:
                    return json.dumps(status['data'], indent=2)
                else:
                    pretty = pprint.pformat(status, indent=2, width=80)
                    logging.info(
                        "{0} in box {1} successful, return is: {2}".format(
                            source, self.args['box'], pretty))
                    return None

    """ Test functions."""

    def create_box(self):
        """ Test to create a box. """
        logging.debug("Creating box: '{0}' on server '{1}'".format(
            self.args['box'], self.args['server']))
        return self.indx.create_box()

    def delete_box(self):
        """ Test to delete a box. """
        logging.debug("Deleting box: '{0}' on server '{1}'".format(
            self.args['box'], self.args['server']))
        return self.indx.delete_box()

    def list_boxes(self):
        """ List the boxes on the INDX server. """
        logging.debug("Listing boxes on server '{0}'".format(
            self.args['server']))
        return self.indx.list_boxes()

    def get_object_ids(self):
        """ Get the IDs of every object in this box. """
        logging.debug(
            "Getting a list of object IDs on server '{0}' in box '{1}'".format(
                self.args['server'], self.args['box']))
        return self.indx.get_object_ids()

    def update(self):
        """ Test to update objects in a box. """
        logging.debug("Updating data to box: '{0}' on server '{1}'".format(
            self.args['box'], self.args['server']))
        #return self.indx.update(self.args['version'], cjson.decode(self.args['data'].read()), all_unicode=True)
        return self.indx.update(self.args['version'],
                                json.loads(self.args['data'].read()))

    def update_json(self):
        """ Test to update json objects in a box. """
        logging.debug("Updating data to box: '{0}' on server '{1}'".format(
            self.args['box'], self.args['server']))
        #return self.indx.update(self.args['version'], cjson.decode(self.args['data'].read()), all_unicode=True)
        return self.indx.update_json(self.args['version'],
                                     json.loads(self.args['data'].read()))

    def update_raw(self):
        """ Test to update raw objects in a box. """
        logging.debug("Updating raw data to box: '{0}' on server '{1}'".format(
            self.args['box'], self.args['server']))
        #return self.indx.update(self.args['version'], cjson.decode(self.args['data'].read()), all_unicode=True)
        return self.indx.update_raw(self.args['version'],
                                    json.loads(self.args['data'].read()))

    def delete(self):
        """ Test to delete objects from a box.
        
            'id' argument should be a list of ids
            e.g., --id=id1 --id=id2
        """
        logging.debug("Deleting data to box: '{0}' on server '{1}'".format(
            self.args['box'], self.args['server']))
        return self.indx.delete(self.args['version'], self.args['id'])

    def get_latest(self):
        """ Get the latest version of every object in this box. """
        logging.debug(
            "Getting latest objects on server '{0}' in box '{1}'".format(
                self.args['server'], self.args['box']))
        return self.indx.get_latest()

    def get_by_ids(self):
        """ Get the latest version of specific objects in this box. """
        logging.debug(
            "Getting latest objects on server '{0}' in box '{1}'".format(
                self.args['server'], self.args['box']))
        ids = self.args['id']
        if type(ids) != type([]):
            ids = [ids]  # put a single id into an array
        return self.indx.get_by_ids(ids)

    def query(self):
        """ Query this box.
             e.g., query="{ '@id': 2983 }"
             or query="{ 'firstname': 'dan' }"           
        """
        logging.debug(
            "Querying server '{0}' in box '{1}' with depth '{2}'".format(
                self.args['server'], self.args['box'], self.args['depth']))
        return self.indx.query(self.args['query'],
                               depth=int(self.args['depth']))

    def diff(self):
        """ Run a diff of two version of this box. """
        logging.debug("Calling diff on server '{0}' in box '{1}'".format(
            self.args['server'], self.args['box']))

        to_version = None
        if "to" in self.args and self.args['to'] is not None:
            to_version = self.args['to']

        return self.indx.diff(self.args['return_objs'],
                              self.args['from'],
                              to_version=to_version)


#    def listen(self):
#        """ Listen to updates to the database (locally, not with HTTP) and print out the diff in realtime. """
#        self.check_args(['box', 'username', 'password'])
#
#        def observer(notify):
#            print "Version updated to: {0}".format(notify.payload)
#
#        def err_cb(failure):
#            logging.error("Error in test listen: {0}".format(failure))
#            reactor.stop()
#
#        def connected_cb(conn):
#            print "Listening..."
#            conns = {"conn": conn}
#            store = ObjectStoreAsync(conns, self.args['username'], self.appid, "127.0.0.1") # TODO get the IP a better way? does it matter here?
#            store.listen(observer)
#
#        d = database.connect_box_raw(self.args['box'], self.args['username'], self.args['password'])
#        d.addCallbacks(connected_cb, err_cb)
#        reactor.run()

    def add_file(self):
        """ Add a file to the database. """
        logging.debug(
            "Adding a file to server '{0}' in box '{1}', with id: {2}".format(
                self.args['server'], self.args['box'], self.args['id']))
        return self.indx.add_file(self.args['version'], self.args['id'],
                                  self.args['data'].read(),
                                  self.args['contenttype'])

    def delete_file(self):
        """ Delete a file from the database. """
        logging.debug(
            "Deleting a file from server '{0}' in box '{1}', with id: {2}".
            format(self.args['server'], self.args['box'], self.args['id']))
        return self.indx.delete_file(self.args['version'], self.args['id'])

    def get_file(self):
        """ Get a file from the database. """
        logging.debug("Calling get_file on server '{0}' in box '{1}'".format(
            self.args['server'], self.args['box']))
        # returns the file to stdout so it can be piped to a file
        print self.indx.get_file(self.args['id'])

    def list_files(self):
        """ Get a list of the files from the database. """
        logging.debug("Calling list_files on server '{0}' in box '{1}'".format(
            self.args['server'], self.args['box']))
        return self.indx.list_files()

    def set_acl(self):
        """ Set an ACL for a target user for a database. """
        logging.debug("Calling set_acl on server '{0}' in box '{1}'".format(
            self.args['server'], self.args['box']))
        return self.indx.set_acl(self.args['acl'],
                                 self.args['target_username'])

    def set_acl_public(self):
        """ Set an ACL for the public uer for a database. """
        logging.debug(
            "Calling set_acl_public on server '{0}' in box '{1}'".format(
                self.args['server'], self.args['box']))
        return self.indx.set_acl_public(self.args['acl'])

    def get_acls(self):
        """ Get ACLs for a database. """
        logging.debug("Calling get_acls on server '{0}' in box '{1}'".format(
            self.args['server'], self.args['box']))
        return self.indx.get_acls()

    def generate_new_key(self):
        """ Generate and store a new key, returning the public and public-hash parts of the key. """
        logging.debug(
            "Calling generate_new_key on server '{0}' in box '{1}'".format(
                self.args['server'], self.args['box']))
        return self.indx.generate_new_key()

    def create_root_box(self):
        """ Create root box for a user. """
        logging.debug(
            "Calling create_root_box on server '{0}' for box '{1}'".format(
                self.args['server'], self.args['box']))
        return self.indx.create_root_box(self.args['box'])

    def create_user(self):
        """ Create a new user. """
        logging.debug(
            "Calling create_user on server '{0}' with target username '{1}'".
            format(self.args['server'], self.args['target_username']))
        return self.indx.create_user(self.args['target_username'],
                                     self.args['target_password'])

    def link_remote_box(self):
        """ Link a remote box with a local box. """
        logging.debug(
            "Calling link_remote_box on remote_address '{0}', remote_box '{1}', remote_token '{2}'"
            .format(self.args['remote_address'], self.args['remote_box'],
                    self.args['remote_token']))
        return self.indx.link_remote_box(self.args['remote_address'],
                                         self.args['remote_box'],
                                         self.args['remote_token'])

    def generate_token(self):
        """ Generate a token for this box, and print it. """
        logging.debug("Calling generate_token on box {0}.".format(
            self.args['box']))
        return_d = Deferred()

        def token_cb(token):
            return_d.callback({"code": 200, "data": token})

        self.auth_and_get_token(True).addCallbacks(token_cb, return_d.errback)
        return return_d
예제 #12
0
파일: client.py 프로젝트: allmende/indx
 def token_cb(token):
     if not self.indx:
         self.indx = IndxClient(self.args['server'], self.args['box'], self.appid, token = token, client = self.client)
     do_call()
예제 #13
0
파일: nike-indx.py 프로젝트: kyonetca/indx
 def token_cb(token):
     indx = IndxClient(server_url, box, appid, token = token)
     return_d.callback(indx)
예제 #14
0
                    help="Earliest date to download in format: 2013-01-20")
parser.add_argument('--debug',
                    default=False,
                    action="store_true",
                    help="Enable debugging")

args = vars(parser.parse_args())
password = getpass.getpass("Nike+ password: "******"INDX password: "******""" Assert data into the INDX.
        If the version is incorrect, the correct version will be grabbed and the update re-sent.

        data -- An object to assert into the box.
    """
    global version
    try:
        response = indx.update(version, data)
        version = response['data']['@version']  # update the version
    except Exception as e:
        if isinstance(
예제 #15
0
class instagramService:
    def __init__(self, config):
        self.config = config
        self.api = InstagramAPI(access_token=config['access_token'])
        self.version = 0
        self.batch = []
        self.batch_users = []
        self.feed_count = 0
        self.feed_count_total = 0
        self.instagram_username = config['instagram_username']
        self.instagram_user_id = config['instagram_user_id']

    def stop_and_exit_service(self):
        logging.debug('instagram Service - HARD QUIT - instagram SERVICE')
        sys.exit()

    def run_main_services(self):

        main_services_d = Deferred()
        try:
            print "running instagram main services"

            def find_followers_cb(res):
                def find_friends_cb(res):
                    def get_authen_user_feed_cb(res):
                        def get_popular_media_cb(res):

                            main_services_d.callback(True)

                        self.get_popular_media().addCallbacks(
                            get_popular_media_cb, lambda failure: logging.
                            error("Update Error {0}".format(failure)))

                    self.get_authenticated_user_feed().addCallbacks(
                        get_authen_user_feed_cb, lambda failure: logging.error(
                            "Update Error {0}".format(failure)))

                self.find_friends(
                    self.config['instagram_user_id']).addCallbacks(
                        find_friends_cb, lambda failure: logging.error(
                            "Update Error{0}".format(failure)))

            self.find_followers(self.config['instagram_user_id']).addCallbacks(
                find_followers_cb, lambda failure: logging.error(
                    "Update Error{0}".format(failure)))
            #self.subscribe_to_objects_by_tag(self.config['instagram_search_words'])
        except:
            logging.debug(
                'Service Instagram - Could not run main service due to error: {0}'
                .format(sys.exc_info()))

        return main_services_d

    def get_indx(self):

        return_d = Deferred()

        def authed_cb():
            logging.debug("in service_tweets - Authed Callback")
            logging.debug(
                "in service_tweets - get_indx authclient Auth status: {0}".
                format(authclient.is_authed))

            def token_cb(token):
                self.indx_con = IndxClient(self.config['address'],
                                           self.config['box'],
                                           appid,
                                           token=token,
                                           client=authclient.client)
                return_d.callback(True)

            authclient.get_token(self.config['box']).addCallbacks(
                token_cb, return_d.errback)

        def authed_cb_fail(re):
            logging.debug(
                "in service tweets - get_indx/authed_cb failed for reason {0}".
                format(re))
            return_d.errback

        logging.debug("in service_instagram - get_indx")
        authclient = IndxClientAuth(self.config['address'], appid)
        authclient.auth_plain(self.config['user'],
                              self.config['password']).addCallbacks(
                                  lambda response: authed_cb(), authed_cb_fail)

        return return_d

    def get_authenticated_user_feed(self):
        auth_d = Deferred()

        def found_cb(results):
            friends_number = 0
            followers_number = 0
            since_id = ""
            #let's see if the object has some nice things in it.
            try:
                config_returned = results['data']['service_instagram_config']
                friends_number = int(
                    config_returned['friends_list_size'][0]['@value'])
                followers_number = int(
                    config_returned['followers_list_size'][0]['@value'])
                since_id = int(config_returned['since_id'][0]['@value'])
                logging.info('Found the instagram Config Object.')
            except:
                #print sys.exc_info()
                pass

            #print "Getting Auth User's feed"
            user_feed = self.api.user_media_feed()[0]
            try:
                latest_id = user_feed[0].id
            except:
                latest_id = "Null"
            ##find the highest id...

            if (latest_id != since_id):

                logging.info("Found some new media, will insert it to INDX")
                since_id = latest_id
                objects_to_insert = []
                current_timestamp = str(
                    time.time()).split(".")[0]  #str(datetime.now())
                timestamp = str(datetime.now().isoformat('T')).split(".")[0]
                current_timestamp = str(datetime.now())

                #the user responsible for this

                #now create some nice user objects...
                for media in user_feed:

                    media_user_id = media.user.id
                    media_username = media.user.username

                    uniq_id = "instagram_media_" + media.id
                    user_feed_obj = {
                        "@id": uniq_id,
                        "app_object": appid,
                        "timestamp": timestamp,
                        "type": "post",
                        "instagram_user_id_indx": "instagram_user_me",
                        "instagram_user_id": media_user_id,
                        "instagram_username": media_username,
                        "media_id": media.id,
                        "media_caption": media.caption,
                        "media_url": media.get_standard_resolution_url(),
                        "media_like_count": media.like_count
                    }

                    #need to add INDX objects for tags and locations and the indx user of this

                    #create location if available
                    try:
                        location = media.location
                        uniq_id = "instagram_location_" + location.id
                        location_obj = {
                            "@id": uniq_id,
                            "app_object": appid,
                            "timestamp": timestamp,
                            "type": "location",
                            "instagram_location_id": location.id,
                            "location_name": location.name,
                            "latitude": location.point.latitude,
                            "longitude": location.point.longitude
                        }

                        #now add this location to the user_feed_obj
                        user_feed_obj['media_location_id'] = location_obj
                        #thhen add it to a list of ojects to insert.
                        objects_to_insert.append(location_obj)
                    except:
                        pass

                    try:
                        tag_ids = []
                        for tag in media.tags:
                            uniq_id = "instagram_tag_" + tag.name
                            tag_obj = {
                                "@id": uniq_id,
                                "app_object": appid,
                                "timestamp": timestamp,
                                "type": "tag",
                                "instagram_tag_name": tag.name
                            }
                            tag_ids.append(uniq_id)
                            objects_to_insert.append(tag_obj)

                        #now add this location to the user_feed_obj
                        user_feed_obj['tags'] = tag_ids
                    except:
                        pass

                    objects_to_insert.append(user_feed_obj)

                #now create the instagram_user_me object
                uniq_id = "instagram_user_me"
                instagram_me_obj = {
                    "@id": uniq_id,
                    "app_object": appid,
                    "timestamp": timestamp,
                    "type": "user",
                    "instagram_user_id": media_user_id,
                    "instagram_username": media_username
                }
                objects_to_insert.append(instagram_me_obj)

                #and create the instagram config
                instagram_config_obj = {
                    "@id": "service_instagram_config",
                    "app_object": appid,
                    "type": "config",
                    "config_last_updated_at": timestamp,
                    "config_for_instagram_user": self.instagram_username,
                    "friends_list_generated_at": timestamp,
                    "follower_list_generated_at": timestamp,
                    "friends_list_size": friends_number,
                    "followers_list_size": followers_number,
                    "since_id": since_id
                }
                objects_to_insert.append(instagram_config_obj)

                def update_cb(re):
                    logging.debug(
                        "network harvest async worked {0}".format(re))
                    auth_d.callback(True)

                def update_cb_fail(re):
                    logging.error(
                        "network harvest async failed {0}".format(re))
                    auth_d.errback

                self.insert_object_to_indx(objects_to_insert).addCallbacks(
                    update_cb, update_cb_fail)
            else:
                logging.info(
                    "already have the latest version of your instagram timeline"
                )
                auth_d.callback(True)

        def error_cb(re):
            found_cb()

        def_search = Deferred()
        find_instagram_config = {"@id": "service_instagram_config"}
        logging.info(
            "Searching for instagram_config to check if Popular feed already harvested... "
        )
        def_search = self.indx_con.query(json.dumps(find_instagram_config))
        def_search.addCallbacks(found_cb, error_cb)

        return auth_d
        # for feed in user_feed:
        #     print feed

    def get_searched_media(self, search_terms):
        print "getting searched media for terms: " + str(search_terms)
        returned_results, page_num = self.api.tag_search(search_terms, 20)
        return returned_results
        # for result in returned_results:
        #     print result

    def get_popular_media(self):
        pop_d = Deferred()

        def found_cb(results):
            friends_number = 0
            followers_number = 0
            since_id = ""
            #let's see if the object has some nice things in it.
            try:
                config_returned = results['data']['service_instagram_config']
                friends_number = int(
                    config_returned['friends_list_size'][0]['@value'])
                followers_number = int(
                    config_returned['followers_list_size'][0]['@value'])
                since_id = int(config_returned['since_id'][0]['@value'])
                logging.info('Found the instagram Config Object.')
            except:
                #print sys.exc_info()
                pass

        #if(since_id > 0):
            logging.info("getting popular media")

            objects_to_insert = []
            current_timestamp = str(
                time.time()).split(".")[0]  #str(datetime.now())
            timestamp = str(datetime.now().isoformat('T')).split(".")[0]
            current_timestamp = str(datetime.now())

            popular_media = self.api.media_popular(count=20)

            for media in popular_media:

                media_user_id = media.user.id
                media_username = media.user.username

                #now create the instagram_user object
                uniq_user_id = "instagram_user_" + media.user.id
                instagram_media_obj = {
                    "@id": uniq_user_id,
                    "app_object": appid,
                    "timestamp": timestamp,
                    "type": "user",
                    "instagram_user_id": media_user_id,
                    "instagram_username": media_username
                }
                objects_to_insert.append(instagram_media_obj)

                uniq_id = "instagram_media_" + media.id
                media_obj = {
                    "@id": uniq_id,
                    "app_object": appid,
                    "timestamp": timestamp,
                    "type": "post",
                    "instagram_user_id_indx": uniq_user_id,
                    "instagram_user_id": media_user_id,
                    "instagram_username": media_username,
                    "media_id": media.id,
                    "media_caption": media.caption,
                    "media_url": media.get_standard_resolution_url(),
                    "media_like_count": media.like_count
                }

                #need to add INDX objects for tags and locations and the indx user of this

                #create location if available
                try:
                    location = media.location
                    uniq_id = "instagram_location_" + location.id
                    location_obj = {
                        "@id": uniq_id,
                        "app_object": appid,
                        "timestamp": timestamp,
                        "type": "location",
                        "instagram_location_id": location.id,
                        "location_name": location.name,
                        "latitude": location.point.latitude,
                        "longitude": location.point.longitude
                    }

                    #now add this location to the user_feed_obj
                    media_obj['media_location_id'] = location_obj
                    #thhen add it to a list of ojects to insert.
                    objects_to_insert.append(location_obj)
                except:
                    pass

                try:
                    tag_ids = []
                    for tag in media.tags:
                        uniq_id = "instagram_tag_" + tag.name
                        tag_obj = {
                            "@id": uniq_id,
                            "app_object": appid,
                            "type": "tag",
                            "timestamp": timestamp,
                            "instagram_tag_name": tag.name
                        }
                        tag_ids.append(uniq_id)
                        objects_to_insert.append(tag_obj)

                    #now add this location to the user_feed_obj
                    media_obj['tags'] = tag_ids
                except:
                    pass

                objects_to_insert.append(media_obj)

            def update_cb(re):
                logging.debug("network harvest async worked {0}".format(re))
                pop_d.callback(True)

            def update_cb_fail(re):
                logging.error("network harvest async failed {0}".format(re))
                pop_d.errback

            #and create the instagram config
            instagram_config_obj = {
                "@id": "service_instagram_config",
                "app_object": appid,
                "type": "config",
                "config_last_updated_at": timestamp,
                "config_for_instagram_user": self.instagram_username,
                "friends_list_generated_at": timestamp,
                "follower_list_generated_at": timestamp,
                "friends_list_size": friends_number,
                "followers_list_size": followers_number,
                "since_id": since_id
            }
            objects_to_insert.append(instagram_config_obj)

            self.insert_object_to_indx(objects_to_insert).addCallbacks(
                update_cb, update_cb_fail)

        def error_cb(re):
            found_cb()

        def_search = Deferred()
        find_instagram_config = {"@id": "service_instagram_config"}
        logging.info(
            "Searching for instagram_config to check if Popular feed already harvested... "
        )
        def_search = self.indx_con.query(json.dumps(find_instagram_config))
        def_search.addCallbacks(found_cb, error_cb)

        return pop_d

    def find_user(self, username):
        data = self.api.user_search(username, count=20)
        print data

    def find_followers(self, userid):

        #first do a lookup and see if the latest set if followers has allready been found
        follower_d = Deferred()

        def found_cb(results):
            friends_number = 0
            followers_number = 0
            since_id = 0
            #let's see if the object has some nice things in it.
            try:
                config_returned = results['data']['service_instagram_config']
                friends_number = int(
                    config_returned['friends_list_size'][0]['@value'])
                followers_number = int(
                    config_returned['followers_list_size'][0]['@value'])
                since_id = int(config_returned['since_id'][0]['@value'])
                logging.info('Found the instagram Config Object.')
            except:
                #print sys.exc_info()
                pass
            followed_by = self.api.user_followed_by(userid)[0]
            #print "followed_by length: "+str(len(followed_by))
            #print str(followed_by)
            #see if the number is different, if it is, then update.. could be nicer than this, but for now, it will work (ish)
            if (len(followed_by) != followers_number) or (followers_number
                                                          == 0):

                followers_number = len(followed_by)
                objects_to_insert = []
                current_timestamp = str(
                    time.time()).split(".")[0]  #str(datetime.now())
                timestamp = str(datetime.now().isoformat('T')).split(".")[0]
                uniq_id = "instagram_follower_network_for_user_me"  #+self.instagram_username
                followed_by_obj = {
                    "@id": uniq_id,
                    "app_object": appid,
                    "instagram_username": self.instagram_username,
                    "instagram_user_id": self.instagram_user_id,
                    "timestamp": timestamp,
                    "followed_by_count": len(followed_by)
                }

                followers_ids = []
                for follower in followed_by:
                    #print follower.username
                    uniq_id = "instagram_user_" + str(follower.username)
                    follower_obj = {
                        "@id": uniq_id,
                        "app_object": appid,
                        "type": "user",
                        "instagram_username": str(follower.username),
                        "timestamp": timestamp
                    }
                    objects_to_insert.append(follower_obj)
                    #we can add this to the followed_by_obj later
                    followers_ids.append(uniq_id)

                #link the followers for me
                followed_by_obj['follower_ids'] = followers_ids

                # print followed_by_objs
                #for friend in friends_list:
                #print friend
                #now append the results
                def update_cb(re):
                    logging.debug(
                        "network harvest async worked {0}".format(re))
                    follower_d.callback(True)

                def update_cb_fail(re):
                    logging.error(
                        "network harvest async failed {0}".format(re))
                    follower_d.errback

                instagram_config_obj = {
                    "@id": "service_instagram_config",
                    "app_object": appid,
                    "type": "config",
                    "config_last_updated_at": timestamp,
                    "config_for_instagram_user": self.instagram_username,
                    "friends_list_generated_at": timestamp,
                    "follower_list_generated_at": timestamp,
                    "friends_list_size": friends_number,
                    "followers_list_size": followers_number,
                    "since_id": since_id
                }

                objects_to_insert.append(followed_by_obj)
                #objects_to_insert.append(followers)
                objects_to_insert.append(instagram_config_obj)

                self.insert_object_to_indx(objects_to_insert).addCallbacks(
                    update_cb, update_cb_fail)
            else:
                follower_d.callback(True)

        def error_cb(re):
            found_cb()

        def_search = Deferred()
        find_instagram_config = {"@id": "service_instagram_config"}
        logging.info(
            "Searching for instagram_config to check if network already harvested... "
        )
        def_search = self.indx_con.query(json.dumps(find_instagram_config))
        def_search.addCallbacks(found_cb, error_cb)

        return follower_d
        # print followed_by

    def find_friends(self, userid):
        friends_d = Deferred()

        def found_cb(results):
            friends_number = 0
            followers_number = 0
            since_id = 0
            #let's see if the object has some nice things in it.
            try:
                config_returned = results['data']['service_instagram_config']
                friends_number = int(
                    config_returned['friends_list_size'][0]['@value'])
                followers_number = int(
                    config_returned['followers_list_size'][0]['@value'])
                since_id = int(config_returned['since_id'][0]['@value'])
                logging.info('Found the instagram Config Object.')
            except:
                #print sys.exc_info()
                pass

            friends_by_list = self.api.user_follows(userid)[0]

            if (len(friends_by_list) != friends_number) or (friends_number
                                                            == 0):

                friends_number = len(friends_by_list)
                objects_to_insert = []
                current_timestamp = str(
                    time.time()).split(".")[0]  #str(datetime.now())
                timestamp = str(datetime.now().isoformat('T')).split(".")[0]
                uniq_id = "instagram_friends_network_for_user_me"  #+self.instagram_username
                friends_by_obj = {
                    "@id": uniq_id,
                    "app_object": appid,
                    "instagram_username": self.instagram_username,
                    "instagram_user_id": self.instagram_user_id,
                    "timestamp": timestamp,
                    "followed_by_count": len(friends_by_list)
                }

                friends_ids = []
                for friend in friends_by_list:
                    #print follower.username
                    uniq_id = "instagram_user_" + str(friend.username)
                    friend_obj = {
                        "@id": uniq_id,
                        "app_object": appid,
                        "type": "user",
                        "instagram_username": str(friend.username),
                        "timestamp": timestamp
                    }
                    objects_to_insert.append(friend_obj)
                    #we can add this to the followed_by_obj later
                    friends_ids.append(uniq_id)

                friends_by_obj['friends_ids'] = friends_ids

                # print friends_by_objs
                #for friend in friends_list:
                #print friend
                #now append the results
                def update_cb(re):
                    logging.debug(
                        "network harvest async worked {0}".format(re))
                    friends_d.callback(True)

                def update_cb_fail(re):
                    logging.error(
                        "network harvest async failed {0}".format(re))
                    friends_d.errback

                instagram_config_obj = {
                    "@id": "service_instagram_config",
                    "app_object": appid,
                    "type": "config",
                    "config_last_updated_at": timestamp,
                    "config_for_instagram_user": self.instagram_username,
                    "friends_list_generated_at": timestamp,
                    "follower_list_generated_at": timestamp,
                    "friends_list_size": friends_number,
                    "followers_list_size": followers_number,
                    "since_id": since_id
                }

                objects_to_insert.append(friends_by_obj)
                #objects_to_insert.append(followers)
                objects_to_insert.append(instagram_config_obj)

                self.insert_object_to_indx(objects_to_insert).addCallbacks(
                    update_cb, update_cb_fail)
            else:
                friends_d.callback(True)

        def error_cb(re):
            found_cb()

        def_search = Deferred()
        find_instagram_config = {"@id": "service_instagram_config"}
        logging.info(
            "Searching for instagram_config to check if network already harvested... "
        )
        def_search = self.indx_con.query(json.dumps(find_instagram_config))
        def_search.addCallbacks(found_cb, error_cb)

        return friends_d
        # print followed_by

    def subscribe_to_objects_by_tag(self, search_terms):
        def process_tag_update(update):
            print update

        reactor = subscriptions.SubscriptionsReactor()
        reactor.register_callback(subscriptions.SubscriptionType.TAG,
                                  process_tag_update)
        self.api.create_subscription(object='tag',
                                     object_id=search_terms,
                                     aspect='media',
                                     callback_url='http://*****:*****@version']
            logging.debug(
                "Succesfully Updated INDX with Objects in update_cb, new diff version of {0}"
                .format(self.version))
            #logging.debug("Inserted Object into INDX: ".format(resp))
            update_d.callback(True)
            #return update_d
            #time.sleep(2)

        def exception_cb(e, service=self, obj=obj):
            logging.debug(
                "Exception Inserting into INDX, probably wrong version given")
            if isinstance(
                    e.value, urllib2.HTTPError
            ):  # handle a version incorrect error, and update the version
                if e.value.code == 409:  # 409 Obsolete
                    response = e.value.read()
                    json_response = json.loads(response)
                    service.version = json_response['@version']
                    logging.debug(
                        'INDX insert error in Instagram Service Object: ' +
                        str(response))
                    try:
                        service.indx_con.update(service.version,
                                                obj).addCallbacks(
                                                    update_cb, exception_cb)
                        #logging.debug('Twitter Service - Successfully added Objects into Box')
                    except:
                        logging.error(
                            'Instagram Service, error on insert {0}'.format(
                                response))
                        update_d.errback(e.value)
                else:
                    logging.error('Instagram Service Unknow error: {0}'.format(
                        e.value.read()))
                    update_d.errback(e.value)
            else:
                logging.error("Error updating INDX: {0}".format(e.value))
                update_d.errback(e.value)

        logging.debug(
            "in Instagram - Trying to insert into indx...Current diff version: {0} and objects (len) given {1}"
            .format(self.version, len(obj)))
        self.indx_con.update(self.version,
                             obj).addCallbacks(update_cb, exception_cb)

        return update_d
예제 #16
0
 def token_cb(token):
     self.indx_con = IndxClient(self.config['address'], self.config['box'], app_id, token = token, client = authclient.client)
     return_d.callback(True)
예제 #17
0
class FacebookService:

    def __init__(self, config):
        logging.debug('Facebook Service - Getting configs from Facebook Controller')
        self.config = config
        self.facebook_access_token_long = config['facebook_access_token_long']
        self.facebook_userid = config['facebook_userid']
        self.facebook_access_token_expire_time = config['facebook_access_token_expire_time']
        self.config_timestamp = config['config_timestamp']
        self.version = 0
        #in reality this is all we need from facebook, but we want to also check that the current date is not over 60 days...
        logging.debug('Facebook Service - Checking if token is not expired')
        self.token_active = False
        if self.check_if_token_not_expired():
            logging.info('Facebook Service - Token is active still. Great News!')
            self.token_active = True

    def get_indx(self):

        return_d = Deferred()

        def authed_cb(): 
            logging.debug("in service_tweets - Authed Callback")
            logging.debug("in service_tweets - get_indx authclient Auth status: {0}".format(authclient.is_authed))
        
            def token_cb(token):
                self.indx_con = IndxClient(self.config['address'], self.config['box'], app_id, token = token, client = authclient.client)
                return_d.callback(True)

            authclient.get_token(self.config['box']).addCallbacks(token_cb, return_d.errback)
         
        def authed_cb_fail(re): 
            logging.debug("in Facebook Service - get_indx/authed_cb failed for reason {0}".format(re))
            return_d.errback   

        logging.debug("in Facebook Service - get_indx")    
        authclient = IndxClientAuth(self.config['address'], app_id)
        authclient.auth_plain(self.config['user'], self.config['password']).addCallbacks(lambda response: authed_cb(), authed_cb_fail)

        return return_d


#     def get_indx(self):
#         return_d = Deferred()
# #        self.indx_con = IndxClient(self.config['address'], self.config['box'], self.config['user'], self.config['password'], app_id)
#         def authed_cb(): 
#             def token_cb(token):
#                 self.indx_con = IndxClient(self.config['address'], self.config['box'], app_id, token = token)
#                 return_d.callback(True)

#             authclient.get_token(self.config['box']).addCallbacks(token_cb, return_d.errback)
            
#         authclient = IndxClientAuth(self.config['address'], app_id)
#         authclient.auth_plain(self.config['user'], self.config['password']).addCallbacks(lambda response: authed_cb(), return_d.errback)
#         return return_d

    def is_token_active(self):
        return self.token_active


    #need to check if the token has already expired, we have around 60 days. Returns true if token still active
    def check_if_token_not_expired(self):
        current_time = datetime.datetime.now()
        access_token_time = (self.config_timestamp).split(".")[0] # we just want it as yyyy-mm-ddT:hh:mm:ss
        access_token_time = self.createTimestamp(access_token_time);
        delta = current_time - access_token_time
        delta_days = delta.days
        if (delta_days) < 60:
            return True
        else:
            return False

    #this should be a library function somewhere...
    def createTimestamp(self,strTimestamp):
        try:
            return datetime.datetime.strptime(strTimestamp, '%Y-%m-%dT%H:%S:%M')
        except:
            if not ":" in str(strTimestamp):
                strTimestamp = strTimestamp+" 00:00:00"
                return datetime.datetime.strptime(strTimestamp, '%Y-%m-%d %H:%S:%M')
            else:
                pass


    def harvest_all(self):

        harvest_d = Deferred()

        if(self.is_token_active()):
            logging.debug("Service Controller Facebook - Running Facebook Service!")

            def havest_friends_cb(re):
                logging.debug("Service Controller Facebook - Updated Profile")

                def havest_statuses_cb(res):
                    logging.debug("Service Controller Facebook - Updated Status {0}".format(res))
                    harvest_d.callback(True)
                    # def havest_profile_cb(resu):
                    #     logging.debug("Service Controller Facebook - Updated Friends")
                    #     #update every hour
                    #     #reactor.callLater(3600.0, loop_harvester);
                    #     harvest_d.callback(True)

                    # self.harvest_facebook_profile().addCallbacks(havest_profile_cb, lambda failure: logging.error("Facebook Service Controller - Callback Failure: Harvest Profile"))

                self.harvest_facebook_statuses().addCallbacks(havest_statuses_cb, lambda failure: logging.error("Facebook Service Controller - Callback Failure: Harvest Statuses"))

            self.harvest_facebook_friends().addCallbacks(havest_friends_cb, lambda failure: logging.error("Facebook Service Controller - Callback Failure: Harvest Friends"))

        return harvest_d


        ##here are all the facebook methods to havest the data
    def harvest_facebook_profile(self):
        harvest_profile_d = Deferred()

        logging.info('Facebook Service - Getting users Facebook Profile')
        graph = GraphAPI(self.facebook_access_token_long)
        profile = graph.get_object("me")
        timestamp = str(datetime.datetime.now().isoformat('T')).split(".")[0]
        uniq_id = "facebook_profile_for_me"
        object_to_insert = {"@id":uniq_id, "app_object": app_id, "timestamp": timestamp, "facebook_profile": profile}

        #now need to perform the asnc
        def insert_cb(re):
            logging.debug("Facebook Service - Found Facebook Profile information, Added To INDX {0} profile items".format(len(profile)))
            harvest_profile_d.callback(True)

        def insert_cb_fail(re):
            harvest_profile_d.callback(True)

        self.insert_object_to_indx(object_to_insert).addCallbacks(insert_cb, insert_cb_fail)

        return harvest_profile_d

    ##here are all the facebook methods to havest the data
    def harvest_facebook_statuses(self):
        harvest_status_d = Deferred()

        def found_cb(results):
            friends_number = 0
            followers_number = 0
            since_id = 0
            #let's see if the object has some nice things in it.
            try:
                config_returned = results['data']['service_facebook_config']
                friends_number = int(config_returned['friends_list_size'][0]['@value'])
                followers_number = int(config_returned['followers_list_size'][0]['@value'])
                since_id = int(config_returned['since_id'][0]['@value'])
                logging.debug('Found the Facebook Config Object.')
            except:
                #print sys.exc_info()
                pass


            logging.info('Facebook Service - Getting users Facebook Statuses')
            graph = GraphAPI(self.facebook_access_token_long)
            profile = graph.get_object("me")
            if len(profile)>1:
                #print "got Profile so should be able to get status"
                facebook_id = str(profile['id'])
                facebook_username = str(profile['name'])
                query = facebook_id#+"/permissions"
                statuses =   graph.get_connections(facebook_id, "statuses", limit=10000) #graph.get_object(query) #
                timestamp = str(datetime.datetime.now().isoformat('T')).split(".")[0]
                statuses = statuses['data']

                #need to make sure that we are not fetching statuses already added...
                latest_status_id = 0
                try:
                    latest_status_id = statuses[0]['id']
                except:
                    pass
                if (latest_status_id > since_id) or (latest_status_id==0):
                    since_id = latest_status_id
                    objects_to_insert = []
                    for status in statuses:

                        uniq_id = "facebook_statuses_"+str(status['id'])
                        facebook_status_obj = {"@id":uniq_id, "app_object": app_id, "timestamp":timestamp, "type": "post", "facebook_status_id":status['id'],
                        "facebook_user_indx": "facebook_user_me", "facebook_user_id":facebook_id}
                        #for each status, get the time
                        facebook_status_obj['facebook_status_text'] = status['message']
                        facebook_status_obj['facebook_status_timestamp'] = str(status['updated_time']).split("+")[0]

                        like_ids = []
                        try:
                            likes = status['likes']['data']
                            totalLikes = len(likes)
                            facebook_status_obj['facebook_status_total_likes'] = str(totalLikes)
                            for like in likes:
                                #construct the facebook user that liked your post
                                uniq_id = "facebook_user_"+str(like['id'])
                                facebook_user_obj = {"@id":uniq_id, "app_object": app_id, "timestamp":timestamp, "type": "user", "facebook_username": like['name'], "facebook_id":like['id']}
                                like_ids.append(uniq_id)
                                objects_to_insert.append(facebook_user_obj)
                        except:
                            pass

                        comment_ids = []
                        try:
                            comments = status['comments']['data']
                            totalcomments = len(comments)
                            facebook_status_obj['facebook_status_total_comments'] = str(totalcomments)
                            for comment in comments:
                                
                                #first create the comment user
                                facebook_comment_obj_id = "facebook_user_"+str(comment['from']['id'])
                                facebook_user_obj = {"@id":facebook_comment_obj_id, "app_object": app_id, "timestamp":timestamp, "type": "user", "facebook_username": comment['from']['name'], "facebook_id":comment['from']['id']}
                                #add it to the indx objects
                                objects_to_insert.append(facebook_user_obj)

                                #create the facebook comment object
                                uniq_id = "facebook_comment_"+comment['id']
                                facebook_comment_obj = {"@id":uniq_id, "app_object": app_id, "timestamp":timestamp, "type": "post", "facebook_user_indx": facebook_comment_obj_id, 
                                "facebook_id":comment['from']['id'], "facebook_username":comment['from']['name']}
                                #for each status, get the time
                                facebook_comment_obj['facebook_comment_text'] = comment['message']
                                facebook_comment_obj['facebook_comment_timestamp'] = str(comment['created_time']).split("+")[0]
                                facebook_comment_obj['like_count'] = comment['like_count']
                                #add it to the indx objects
                                objects_to_insert.append(facebook_comment_obj)
                                #for reference
                                comment_ids.append(uniq_id)
                        except:
                            pass
                        

                        facebook_status_obj['facebook_like_user_ids_indx'] = like_ids
                        facebook_status_obj['facebook_comment_user_ids_indx'] = comment_ids
                        #add it to the indx objects
                        objects_to_insert.append(facebook_status_obj)

                    #then create the user object
                    uniq_id = "facebook_user_me"
                    facebook_user_obj = {"@id":uniq_id, "app_object": app_id, "timestamp":timestamp, "type": "user", "facebook_username": facebook_username, "facebook_id":facebook_id}
                    objects_to_insert.append(facebook_user_obj)
                    #then add all of the extra info
                
                #and create the instagram config
                facebook_config_obj = {"@id": "service_facebook_config", "app_object": app_id, "type":"config", "config_last_updated_at": timestamp,
                "config_for_facebook_user_id": self.facebook_userid, "friends_list_generated_at": timestamp, "follower_list_generated_at": timestamp,
                "friends_list_size":friends_number, "followers_list_size": followers_number, "since_id":since_id}
                
                objects_to_insert.append(facebook_config_obj)


                #now need to perform the asnc
                def insert_cb(re):
                    logging.info("Facebook Service - Found {0} statuses, Added To INDX".format(len(statuses)))
                    harvest_status_d.callback(True)

                def insert_cb_fail(re):
                    harvest_status_d.callback(True)

                logging.debug("inserting statuses into indx")
                self.insert_object_to_indx(objects_to_insert).addCallbacks(insert_cb, insert_cb_fail)
            else:
                harvest_status_d.callback(True)

        def error_cb(re):
            found_cb()

        def_search = Deferred()
        find_instagram_config = {"@id": "service_facebook_config"} 
        logging.info("Searching for facebook_config to check if latest statues already harvested... ")
        def_search = self.indx_con.query(json.dumps(find_instagram_config))
        def_search.addCallbacks(found_cb, error_cb) 
        
        return harvest_status_d


    def harvest_facebook_friends(self):
        harvest_friends_d = Deferred()

        def found_cb(results):
            friends_number = 0
            followers_number = 0
            since_id = 0
            #let's see if the object has some nice things in it.
            try:
                config_returned = results['data']['service_facebook_config']
                friends_number = int(config_returned['friends_list_size'][0]['@value'])
                followers_number = int(config_returned['followers_list_size'][0]['@value'])
                since_id = int(config_returned['since_id'][0]['@value'])
                logging.debug('Found the Facebook Config Object.')
            except:
                #print sys.exc_info()
                pass

            logging.info('Facebook Service - Getting users Facebook Friends')
            graph = GraphAPI(self.facebook_access_token_long)
            #profile = graph.get_object("me")
            #rint profile
            friends_all = graph.get_connections("me", "friends", limit=1000)
            friends = friends_all['data']
            friends_number = len(friends)
            logging.debug('Facebook Service - Got users Facebook Friends: {0}'.format(friends_number))
            objects_to_insert = []
            timestamp = str(datetime.datetime.now().isoformat('T')).split(".")[0]
            friend_ids = []
            counter = 0
            friends_objs_to_insert = []
            for friend in friends:
                uniq_id = "facebook_user_"+str(friend['id'])
                friend_obj = {"@id":uniq_id, "app_object": app_id, "timestamp":timestamp, "type": "user", "facebook_user_id": friend['id'], "facebook_username": friend['name']}
                friend_ids.append(uniq_id)
                friends_objs_to_insert.append(friend_obj)

            try:
                #now create the me facebook user friend object
                profile = graph.get_object("me")
                facebook_id = str(profile['id'])
                facebook_username = str(profile['name'])
                uniq_id = "facebook_friends_for_user_me"
                facebook_user_friend_obj = {"@id":uniq_id, "app_object": app_id, "timestamp":timestamp, "faceboob_user_id_indx": "facebook_user_me",
                "facebook_user_id": facebook_id, "facebook_username": facebook_username, "facebook_friends_ids_indx": friend_ids}
                objects_to_insert.append(facebook_user_friend_obj)
            except:
                pass

            #now set the config
            facebook_config_obj = {"@id": "service_facebook_config", "app_object": app_id, "type":"config", "config_last_updated_at": timestamp,
            "config_for_facebook_user_id": facebook_id, "friends_list_generated_at": timestamp, "follower_list_generated_at": timestamp,
            "friends_list_size":friends_number, "followers_list_size": followers_number, "since_id":since_id}
            objects_to_insert.append(facebook_config_obj)

            def insertfriends(self):
                if (len(friends_objs_to_insert) > 0):

                    def insert_friend_cb(re):
                        insertfriends(self)

                    def insert_friend_cb_fail(re):
                        insertfriends(self)

                    friends_to_add = []
                    pop_point = 0
                    for x in range(0, 100):
                        try:
                            friends_to_add.append(friends_objs_to_insert[x])
                            pop_point += 1
                        except:
                            pass
                    for x in range(0, pop_point):
                        try:
                            friends_objs_to_insert.pop(0)
                        except:
                            pass
                    #friend_object = friends_objs_to_insert[0]
                    #print "friends list length: "+str(len(friends_objs_to_insert))
                    self.insert_object_to_indx(friends_to_add).addCallbacks(insert_friend_cb, insert_friend_cb_fail)
                else:
                    logging.info("All Facebook Friends Added to Indx, will now proceed.")

            #this is a trick as indxclient is a bit nasty with big inserts
            logging.debug("Inserting friends into indx")
            insertfriends(self)

            #now need to perform the asnc
            def insert_cb(re):
                logging.debug("Facebook Service - Found Friends List, Added To INDX {0} Friends".format(len(friends)))
                harvest_friends_d.callback(True)

            def insert_cb_fail(re):
                harvest_friends_d.callback(True)

            logging.debug("inserting friends config into indx")
            self.insert_object_to_indx(objects_to_insert).addCallbacks(insert_cb, insert_cb_fail)

        def error_cb(re):
            found_cb()

        def_search = Deferred()
        find_facebook_config = {"@id": "service_facebook_config"} 
        logging.info("Searching for facebook_config to check if Popular feed already harvested... ")
        def_search = self.indx_con.query(json.dumps(find_facebook_config))
        def_search.addCallbacks(found_cb, error_cb) 
        
        return harvest_friends_d

    
    def insert_object_to_indx(self, obj):

        update_d = Deferred()

        def update_cb(resp):
            self.version = resp['data']['@version']
            logging.debug("Succesfully Updated INDX with Objects in update_cb, new diff version of {0}".format(self.version))
            #logging.debug("Inserted Object into INDX: ".format(resp))
            update_d.callback(True)
            #return update_d
            #time.sleep(2)

        def exception_cb(e, service=self, obj=obj):
            logging.debug("Exception Inserting into INDX, probably wrong version given")
            if isinstance(e.value, urllib2.HTTPError): # handle a version incorrect error, and update the version
                if e.value.code == 409: # 409 Obsolete
                    response = e.value.read()
                    json_response = json.loads(response)
                    service.version = json_response['@version']
                    logging.debug('INDX insert error in Instagram Service Object: '+str(response))
                    try:
                        service.indx_con.update(service.version, obj).addCallbacks(update_cb, exception_cb)
                        #logging.debug('Twitter Service - Successfully added Objects into Box')
                    except:
                        logging.debug('Instagram Service, error on insert {0}'.format(response))
                        update_d.errback(e.value)
                else:
                    logging.debug('Instagram Service Unknow error: {0}'.format(e.value.read()))
                    update_d.errback(e.value)
            else:
                logging.error("Error updating INDX: {0}".format(e.value))
                update_d.errback(e.value)
        
        logging.debug("in Instagram - Trying to insert into indx...Current diff version: {0} and objects (len) given {1}".format(self.version, len(obj)))
        self.indx_con.update(self.version, obj).addCallbacks(update_cb, exception_cb)
        
        return update_d
예제 #18
0
class FacebookService:
    def __init__(self, config):
        logging.debug(
            'Facebook Service - Getting configs from Facebook Controller')
        self.config = config
        self.facebook_access_token_long = config['facebook_access_token_long']
        self.facebook_userid = config['facebook_userid']
        self.facebook_access_token_expire_time = config[
            'facebook_access_token_expire_time']
        self.config_timestamp = config['config_timestamp']
        self.version = 0
        #in reality this is all we need from facebook, but we want to also check that the current date is not over 60 days...
        logging.debug('Facebook Service - Checking if token is not expired')
        self.token_active = False
        if self.check_if_token_not_expired():
            logging.info(
                'Facebook Service - Token is active still. Great News!')
            self.token_active = True

    def get_indx(self):

        return_d = Deferred()

        def authed_cb():
            logging.debug("in service_tweets - Authed Callback")
            logging.debug(
                "in service_tweets - get_indx authclient Auth status: {0}".
                format(authclient.is_authed))

            def token_cb(token):
                self.indx_con = IndxClient(self.config['address'],
                                           self.config['box'],
                                           app_id,
                                           token=token,
                                           client=authclient.client)
                return_d.callback(True)

            authclient.get_token(self.config['box']).addCallbacks(
                token_cb, return_d.errback)

        def authed_cb_fail(re):
            logging.debug(
                "in Facebook Service - get_indx/authed_cb failed for reason {0}"
                .format(re))
            return_d.errback

        logging.debug("in Facebook Service - get_indx")
        authclient = IndxClientAuth(self.config['address'], app_id)
        authclient.auth_plain(self.config['user'],
                              self.config['password']).addCallbacks(
                                  lambda response: authed_cb(), authed_cb_fail)

        return return_d

#     def get_indx(self):
#         return_d = Deferred()
# #        self.indx_con = IndxClient(self.config['address'], self.config['box'], self.config['user'], self.config['password'], app_id)
#         def authed_cb():
#             def token_cb(token):
#                 self.indx_con = IndxClient(self.config['address'], self.config['box'], app_id, token = token)
#                 return_d.callback(True)

#             authclient.get_token(self.config['box']).addCallbacks(token_cb, return_d.errback)

#         authclient = IndxClientAuth(self.config['address'], app_id)
#         authclient.auth_plain(self.config['user'], self.config['password']).addCallbacks(lambda response: authed_cb(), return_d.errback)
#         return return_d

    def is_token_active(self):
        return self.token_active

    #need to check if the token has already expired, we have around 60 days. Returns true if token still active
    def check_if_token_not_expired(self):
        current_time = datetime.datetime.now()
        access_token_time = (self.config_timestamp).split(".")[
            0]  # we just want it as yyyy-mm-ddT:hh:mm:ss
        access_token_time = self.createTimestamp(access_token_time)
        delta = current_time - access_token_time
        delta_days = delta.days
        if (delta_days) < 60:
            return True
        else:
            return False

    #this should be a library function somewhere...
    def createTimestamp(self, strTimestamp):
        try:
            return datetime.datetime.strptime(strTimestamp,
                                              '%Y-%m-%dT%H:%S:%M')
        except:
            if not ":" in str(strTimestamp):
                strTimestamp = strTimestamp + " 00:00:00"
                return datetime.datetime.strptime(strTimestamp,
                                                  '%Y-%m-%d %H:%S:%M')
            else:
                pass

    def harvest_all(self):

        harvest_d = Deferred()

        if (self.is_token_active()):
            logging.debug(
                "Service Controller Facebook - Running Facebook Service!")

            def havest_friends_cb(re):
                logging.debug("Service Controller Facebook - Updated Profile")

                def havest_statuses_cb(res):
                    logging.debug(
                        "Service Controller Facebook - Updated Status {0}".
                        format(res))
                    harvest_d.callback(True)
                    # def havest_profile_cb(resu):
                    #     logging.debug("Service Controller Facebook - Updated Friends")
                    #     #update every hour
                    #     #reactor.callLater(3600.0, loop_harvester);
                    #     harvest_d.callback(True)

                    # self.harvest_facebook_profile().addCallbacks(havest_profile_cb, lambda failure: logging.error("Facebook Service Controller - Callback Failure: Harvest Profile"))

                self.harvest_facebook_statuses().addCallbacks(
                    havest_statuses_cb, lambda failure: logging.error(
                        "Facebook Service Controller - Callback Failure: Harvest Statuses"
                    ))

            self.harvest_facebook_friends().addCallbacks(
                havest_friends_cb, lambda failure: logging.error(
                    "Facebook Service Controller - Callback Failure: Harvest Friends"
                ))

        return harvest_d

        ##here are all the facebook methods to havest the data
    def harvest_facebook_profile(self):
        harvest_profile_d = Deferred()

        logging.info('Facebook Service - Getting users Facebook Profile')
        graph = GraphAPI(self.facebook_access_token_long)
        profile = graph.get_object("me")
        timestamp = str(datetime.datetime.now().isoformat('T')).split(".")[0]
        uniq_id = "facebook_profile_for_me"
        object_to_insert = {
            "@id": uniq_id,
            "app_object": app_id,
            "timestamp": timestamp,
            "facebook_profile": profile
        }

        #now need to perform the asnc
        def insert_cb(re):
            logging.debug(
                "Facebook Service - Found Facebook Profile information, Added To INDX {0} profile items"
                .format(len(profile)))
            harvest_profile_d.callback(True)

        def insert_cb_fail(re):
            harvest_profile_d.callback(True)

        self.insert_object_to_indx(object_to_insert).addCallbacks(
            insert_cb, insert_cb_fail)

        return harvest_profile_d

    ##here are all the facebook methods to havest the data
    def harvest_facebook_statuses(self):
        harvest_status_d = Deferred()

        def found_cb(results):
            friends_number = 0
            followers_number = 0
            since_id = 0
            #let's see if the object has some nice things in it.
            try:
                config_returned = results['data']['service_facebook_config']
                friends_number = int(
                    config_returned['friends_list_size'][0]['@value'])
                followers_number = int(
                    config_returned['followers_list_size'][0]['@value'])
                since_id = int(config_returned['since_id'][0]['@value'])
                logging.debug('Found the Facebook Config Object.')
            except:
                #print sys.exc_info()
                pass

            logging.info('Facebook Service - Getting users Facebook Statuses')
            graph = GraphAPI(self.facebook_access_token_long)
            profile = graph.get_object("me")
            if len(profile) > 1:
                #print "got Profile so should be able to get status"
                facebook_id = str(profile['id'])
                facebook_username = str(profile['name'])
                query = facebook_id  #+"/permissions"
                statuses = graph.get_connections(
                    facebook_id, "statuses",
                    limit=10000)  #graph.get_object(query) #
                timestamp = str(
                    datetime.datetime.now().isoformat('T')).split(".")[0]
                statuses = statuses['data']

                #need to make sure that we are not fetching statuses already added...
                latest_status_id = 0
                try:
                    latest_status_id = statuses[0]['id']
                except:
                    pass
                if (latest_status_id > since_id) or (latest_status_id == 0):
                    since_id = latest_status_id
                    objects_to_insert = []
                    for status in statuses:

                        uniq_id = "facebook_statuses_" + str(status['id'])
                        facebook_status_obj = {
                            "@id": uniq_id,
                            "app_object": app_id,
                            "timestamp": timestamp,
                            "type": "post",
                            "facebook_status_id": status['id'],
                            "facebook_user_indx": "facebook_user_me",
                            "facebook_user_id": facebook_id
                        }
                        #for each status, get the time
                        facebook_status_obj['facebook_status_text'] = status[
                            'message']
                        facebook_status_obj['facebook_status_timestamp'] = str(
                            status['updated_time']).split("+")[0]

                        like_ids = []
                        try:
                            likes = status['likes']['data']
                            totalLikes = len(likes)
                            facebook_status_obj[
                                'facebook_status_total_likes'] = str(
                                    totalLikes)
                            for like in likes:
                                #construct the facebook user that liked your post
                                uniq_id = "facebook_user_" + str(like['id'])
                                facebook_user_obj = {
                                    "@id": uniq_id,
                                    "app_object": app_id,
                                    "timestamp": timestamp,
                                    "type": "user",
                                    "facebook_username": like['name'],
                                    "facebook_id": like['id']
                                }
                                like_ids.append(uniq_id)
                                objects_to_insert.append(facebook_user_obj)
                        except:
                            pass

                        comment_ids = []
                        try:
                            comments = status['comments']['data']
                            totalcomments = len(comments)
                            facebook_status_obj[
                                'facebook_status_total_comments'] = str(
                                    totalcomments)
                            for comment in comments:

                                #first create the comment user
                                facebook_comment_obj_id = "facebook_user_" + str(
                                    comment['from']['id'])
                                facebook_user_obj = {
                                    "@id": facebook_comment_obj_id,
                                    "app_object": app_id,
                                    "timestamp": timestamp,
                                    "type": "user",
                                    "facebook_username":
                                    comment['from']['name'],
                                    "facebook_id": comment['from']['id']
                                }
                                #add it to the indx objects
                                objects_to_insert.append(facebook_user_obj)

                                #create the facebook comment object
                                uniq_id = "facebook_comment_" + comment['id']
                                facebook_comment_obj = {
                                    "@id": uniq_id,
                                    "app_object": app_id,
                                    "timestamp": timestamp,
                                    "type": "post",
                                    "facebook_user_indx":
                                    facebook_comment_obj_id,
                                    "facebook_id": comment['from']['id'],
                                    "facebook_username":
                                    comment['from']['name']
                                }
                                #for each status, get the time
                                facebook_comment_obj[
                                    'facebook_comment_text'] = comment[
                                        'message']
                                facebook_comment_obj[
                                    'facebook_comment_timestamp'] = str(
                                        comment['created_time']).split("+")[0]
                                facebook_comment_obj['like_count'] = comment[
                                    'like_count']
                                #add it to the indx objects
                                objects_to_insert.append(facebook_comment_obj)
                                #for reference
                                comment_ids.append(uniq_id)
                        except:
                            pass

                        facebook_status_obj[
                            'facebook_like_user_ids_indx'] = like_ids
                        facebook_status_obj[
                            'facebook_comment_user_ids_indx'] = comment_ids
                        #add it to the indx objects
                        objects_to_insert.append(facebook_status_obj)

                    #then create the user object
                    uniq_id = "facebook_user_me"
                    facebook_user_obj = {
                        "@id": uniq_id,
                        "app_object": app_id,
                        "timestamp": timestamp,
                        "type": "user",
                        "facebook_username": facebook_username,
                        "facebook_id": facebook_id
                    }
                    objects_to_insert.append(facebook_user_obj)
                    #then add all of the extra info

                #and create the instagram config
                facebook_config_obj = {
                    "@id": "service_facebook_config",
                    "app_object": app_id,
                    "type": "config",
                    "config_last_updated_at": timestamp,
                    "config_for_facebook_user_id": self.facebook_userid,
                    "friends_list_generated_at": timestamp,
                    "follower_list_generated_at": timestamp,
                    "friends_list_size": friends_number,
                    "followers_list_size": followers_number,
                    "since_id": since_id
                }

                objects_to_insert.append(facebook_config_obj)

                #now need to perform the asnc
                def insert_cb(re):
                    logging.info(
                        "Facebook Service - Found {0} statuses, Added To INDX".
                        format(len(statuses)))
                    harvest_status_d.callback(True)

                def insert_cb_fail(re):
                    harvest_status_d.callback(True)

                logging.debug("inserting statuses into indx")
                self.insert_object_to_indx(objects_to_insert).addCallbacks(
                    insert_cb, insert_cb_fail)
            else:
                harvest_status_d.callback(True)

        def error_cb(re):
            found_cb()

        def_search = Deferred()
        find_instagram_config = {"@id": "service_facebook_config"}
        logging.info(
            "Searching for facebook_config to check if latest statues already harvested... "
        )
        def_search = self.indx_con.query(json.dumps(find_instagram_config))
        def_search.addCallbacks(found_cb, error_cb)

        return harvest_status_d

    def harvest_facebook_friends(self):
        harvest_friends_d = Deferred()

        def found_cb(results):
            friends_number = 0
            followers_number = 0
            since_id = 0
            #let's see if the object has some nice things in it.
            try:
                config_returned = results['data']['service_facebook_config']
                friends_number = int(
                    config_returned['friends_list_size'][0]['@value'])
                followers_number = int(
                    config_returned['followers_list_size'][0]['@value'])
                since_id = int(config_returned['since_id'][0]['@value'])
                logging.debug('Found the Facebook Config Object.')
            except:
                #print sys.exc_info()
                pass

            logging.info('Facebook Service - Getting users Facebook Friends')
            graph = GraphAPI(self.facebook_access_token_long)
            #profile = graph.get_object("me")
            #rint profile
            friends_all = graph.get_connections("me", "friends", limit=1000)
            friends = friends_all['data']
            friends_number = len(friends)
            logging.debug(
                'Facebook Service - Got users Facebook Friends: {0}'.format(
                    friends_number))
            objects_to_insert = []
            timestamp = str(
                datetime.datetime.now().isoformat('T')).split(".")[0]
            friend_ids = []
            counter = 0
            friends_objs_to_insert = []
            for friend in friends:
                uniq_id = "facebook_user_" + str(friend['id'])
                friend_obj = {
                    "@id": uniq_id,
                    "app_object": app_id,
                    "timestamp": timestamp,
                    "type": "user",
                    "facebook_user_id": friend['id'],
                    "facebook_username": friend['name']
                }
                friend_ids.append(uniq_id)
                friends_objs_to_insert.append(friend_obj)

            try:
                #now create the me facebook user friend object
                profile = graph.get_object("me")
                facebook_id = str(profile['id'])
                facebook_username = str(profile['name'])
                uniq_id = "facebook_friends_for_user_me"
                facebook_user_friend_obj = {
                    "@id": uniq_id,
                    "app_object": app_id,
                    "timestamp": timestamp,
                    "faceboob_user_id_indx": "facebook_user_me",
                    "facebook_user_id": facebook_id,
                    "facebook_username": facebook_username,
                    "facebook_friends_ids_indx": friend_ids
                }
                objects_to_insert.append(facebook_user_friend_obj)
            except:
                pass

            #now set the config
            facebook_config_obj = {
                "@id": "service_facebook_config",
                "app_object": app_id,
                "type": "config",
                "config_last_updated_at": timestamp,
                "config_for_facebook_user_id": facebook_id,
                "friends_list_generated_at": timestamp,
                "follower_list_generated_at": timestamp,
                "friends_list_size": friends_number,
                "followers_list_size": followers_number,
                "since_id": since_id
            }
            objects_to_insert.append(facebook_config_obj)

            def insertfriends(self):
                if (len(friends_objs_to_insert) > 0):

                    def insert_friend_cb(re):
                        insertfriends(self)

                    def insert_friend_cb_fail(re):
                        insertfriends(self)

                    friends_to_add = []
                    pop_point = 0
                    for x in range(0, 100):
                        try:
                            friends_to_add.append(friends_objs_to_insert[x])
                            pop_point += 1
                        except:
                            pass
                    for x in range(0, pop_point):
                        try:
                            friends_objs_to_insert.pop(0)
                        except:
                            pass
                    #friend_object = friends_objs_to_insert[0]
                    #print "friends list length: "+str(len(friends_objs_to_insert))
                    self.insert_object_to_indx(friends_to_add).addCallbacks(
                        insert_friend_cb, insert_friend_cb_fail)
                else:
                    logging.info(
                        "All Facebook Friends Added to Indx, will now proceed."
                    )

            #this is a trick as indxclient is a bit nasty with big inserts
            logging.debug("Inserting friends into indx")
            insertfriends(self)

            #now need to perform the asnc
            def insert_cb(re):
                logging.debug(
                    "Facebook Service - Found Friends List, Added To INDX {0} Friends"
                    .format(len(friends)))
                harvest_friends_d.callback(True)

            def insert_cb_fail(re):
                harvest_friends_d.callback(True)

            logging.debug("inserting friends config into indx")
            self.insert_object_to_indx(objects_to_insert).addCallbacks(
                insert_cb, insert_cb_fail)

        def error_cb(re):
            found_cb()

        def_search = Deferred()
        find_facebook_config = {"@id": "service_facebook_config"}
        logging.info(
            "Searching for facebook_config to check if Popular feed already harvested... "
        )
        def_search = self.indx_con.query(json.dumps(find_facebook_config))
        def_search.addCallbacks(found_cb, error_cb)

        return harvest_friends_d

    def insert_object_to_indx(self, obj):

        update_d = Deferred()

        def update_cb(resp):
            self.version = resp['data']['@version']
            logging.debug(
                "Succesfully Updated INDX with Objects in update_cb, new diff version of {0}"
                .format(self.version))
            #logging.debug("Inserted Object into INDX: ".format(resp))
            update_d.callback(True)
            #return update_d
            #time.sleep(2)

        def exception_cb(e, service=self, obj=obj):
            logging.debug(
                "Exception Inserting into INDX, probably wrong version given")
            if isinstance(
                    e.value, urllib2.HTTPError
            ):  # handle a version incorrect error, and update the version
                if e.value.code == 409:  # 409 Obsolete
                    response = e.value.read()
                    json_response = json.loads(response)
                    service.version = json_response['@version']
                    logging.debug(
                        'INDX insert error in Instagram Service Object: ' +
                        str(response))
                    try:
                        service.indx_con.update(service.version,
                                                obj).addCallbacks(
                                                    update_cb, exception_cb)
                        #logging.debug('Twitter Service - Successfully added Objects into Box')
                    except:
                        logging.debug(
                            'Instagram Service, error on insert {0}'.format(
                                response))
                        update_d.errback(e.value)
                else:
                    logging.debug('Instagram Service Unknow error: {0}'.format(
                        e.value.read()))
                    update_d.errback(e.value)
            else:
                logging.error("Error updating INDX: {0}".format(e.value))
                update_d.errback(e.value)

        logging.debug(
            "in Instagram - Trying to insert into indx...Current diff version: {0} and objects (len) given {1}"
            .format(self.version, len(obj)))
        self.indx_con.update(self.version,
                             obj).addCallbacks(update_cb, exception_cb)

        return update_d
예제 #19
0
파일: client.py 프로젝트: allmende/indx
class CLIClient:

    def __init__(self, appid = "INDX CLIClient"):
        """ Associate command-line actions with functions, and enforce required variables. """
        self.actions = {'create_box': {'f': self.create_box, 'args': ['box']},
                      'delete_box': {'f': self.delete_box, 'args': ['box']},
                      'list_boxes': {'f': self.list_boxes, 'args': []},
                      'get_object_ids': {'f': self.get_object_ids, 'args': ['box']},
                      'create_user': {'f': self.create_user, 'args': ['target_username','target_password']},
                      'update': {'f': self.update, 'args': ['box','data','version']},
                      'delete': {'f': self.delete, 'args': ['box','id','version']},
                      'get_latest': {'f': self.get_latest, 'args': ['box']},
                      'get_by_ids': {'f': self.get_by_ids, 'args': ['box','id']},
                      'query': {'f': self.query, 'args': ['box','query']},
                      'diff': {'f': self.diff, 'args': ['box','from','return_objs']},
                      #'listen': {'f': self.listen, 'args': ['box']},
                      'add_file': {'f': self.add_file, 'args': ['box','data','id','version','contenttype']},
                      'delete_file': {'f': self.delete_file, 'args': ['box','id','version']},
                      'get_file': {'f': self.get_file, 'args': ['box','id']},
                      'list_files': {'f': self.list_files, 'args': ['box']},
                      'set_acl': {'f': self.set_acl, 'args': ['box','acl','target_username']},
                      'get_acls': {'f': self.get_acls, 'args': ['box']},
                      'generate_new_key': {'f': self.generate_new_key, 'args': ['box']},
                      'create_root_box': {'f': self.create_root_box, 'args': ['box']},
                      'link_remote_box': {'f': self.link_remote_box, 'args': ['box', 'remote_token', 'remote_box', 'remote_address']},
                      'generate_token': {'f': self.generate_token, 'args': ['box']},
                     }

        self.token = None
        self.appid = appid
        self.indx = None


    def set_args(self, args):
        """ Move relevant command-line arguments into local variable. """
        logging.debug("Set args: {0}".format(args))
        self.args = args

        """ Flatten single value lists into flat key/value pair. """
        for key in args:
            if type(args[key]) == type([]) and len(args[key]) == 1:
                args[key] = args[key][0]

        """ Ensure self.server always ends in a / """
        if self.args['server'][-1:] != "/":
            self.args['server'] += "/"


    def check_args(self, required):
        not_set = []
        for key in required:
            if key not in self.args or self.args[key] is None or self.args[key] == "":
                not_set.append(key)
       
        if not self.args['allowempty'] and len(not_set) > 0:
            raise Exception("The following values cannot be empty for this action: {0}".format(", ".join(not_set)))

    def auth_and_get_token(self, get_token):
        """ Authenticate, get a token and call it back to the deferred. """
        return_d = Deferred()

        def authed_cb(): 
            def token_cb(token):
                if token is not None:
                    self.token = token
                    return_d.callback(token)
                else:
                    return_d.callback(None)

            if get_token:
                authclient.get_token(self.args['box']).addCallbacks(token_cb, return_d.errback)
            else:
                token_cb(None)
            
        authclient = IndxClientAuth(self.args['server'], self.appid)
        self.client = authclient.client
        authclient.auth_plain(self.args['username'], self.args['password']).addCallbacks(lambda response: authed_cb(), return_d.errback)

        return return_d

    def call_action(self, name, *args, **kwargs):
        """ Calls an action by name. """
        return_d = Deferred()

        action = self.actions[name]
        f = action['f']
        self.check_args(action['args'])

        def do_call():
            f(*args, **kwargs).addCallbacks(lambda status: return_d.callback(self.parse_status(name, status)), return_d.errback)

        if not self.token:
            def token_cb(token):
                if not self.indx:
                    self.indx = IndxClient(self.args['server'], self.args['box'], self.appid, token = token, client = self.client)
                do_call()

            self.auth_and_get_token(IndxClient.requires_token(f)).addCallbacks(token_cb, return_d.errback)
        else:
            do_call()
            
        return return_d


    def parse_status(self, source, status):
        """ Parse the status returned from the server, and raise an Exception if necessary. """

        if status is not None: # status is None when a file has been printed raw
            if status['code'] < 200 or status['code'] > 299:
                raise Exception("{0} in box {1} failed. Response is {2} with code {3}".format(source, self.args['box'], status['message'], status['code']))
            else:
                if "data" in status and self.args['jsondata']:
                    return json.dumps(status['data'], indent = 2)
                else:
                    pretty = pprint.pformat(status, indent=2, width=80)
                    logging.info("{0} in box {1} successful, return is: {2}".format(source, self.args['box'], pretty))
                    return None


    """ Test functions."""

    def create_box(self):
        """ Test to create a box. """
        logging.debug("Creating box: '{0}' on server '{1}'".format(self.args['box'], self.args['server']))
        return self.indx.create_box()

    def delete_box(self):
        """ Test to delete a box. """
        logging.debug("Deleting box: '{0}' on server '{1}'".format(self.args['box'], self.args['server']))
        return self.indx.delete_box()

    def list_boxes(self):
        """ List the boxes on the INDX server. """
        logging.debug("Listing boxes on server '{0}'".format(self.args['server']))
        return self.indx.list_boxes()


    def get_object_ids(self):
        """ Get the IDs of every object in this box. """
        logging.debug("Getting a list of object IDs on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
        return self.indx.get_object_ids()


    def update(self):
        """ Test to update objects in a box. """
        logging.debug("Updating data to box: '{0}' on server '{1}'".format(self.args['box'], self.args['server']))
        #return self.indx.update(self.args['version'], cjson.decode(self.args['data'].read()), all_unicode=True)
        return self.indx.update(self.args['version'], json.loads(self.args['data'].read()))


    def delete(self):
        """ Test to delete objects from a box.
        
            'id' argument should be a list of ids
            e.g., --id=id1 --id=id2
        """
        logging.debug("Deleting data to box: '{0}' on server '{1}'".format(self.args['box'], self.args['server']))
        return self.indx.delete(self.args['version'], self.args['id'])


    def get_latest(self):
        """ Get the latest version of every object in this box. """
        logging.debug("Getting latest objects on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
        return self.indx.get_latest()


    def get_by_ids(self):
        """ Get the latest version of specific objects in this box. """
        logging.debug("Getting latest objects on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
        ids = self.args['id']
        if type(ids) != type([]):
            ids = [ids] # put a single id into an array
        return self.indx.get_by_ids(ids)


    def query(self):
        """ Query this box.
             e.g., query="{ '@id': 2983 }"
             or query="{ 'firstname': 'dan' }"           
        """
        logging.debug("Querying server '{0}' in box '{1}' with depth '{2}'".format(self.args['server'], self.args['box'], self.args['depth']))
        return self.indx.query(self.args['query'], depth = int(self.args['depth']))


    def diff(self):
        """ Run a diff of two version of this box. """
        logging.debug("Calling diff on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))

        to_version = None
        if "to" in self.args and self.args['to'] is not None:
            to_version = self.args['to']

        return self.indx.diff(self.args['return_objs'], self.args['from'], to_version = to_version)


#    def listen(self):
#        """ Listen to updates to the database (locally, not with HTTP) and print out the diff in realtime. """
#        self.check_args(['box', 'username', 'password'])
#
#        def observer(notify):
#            print "Version updated to: {0}".format(notify.payload)
#
#        def err_cb(failure):
#            logging.error("Error in test listen: {0}".format(failure))
#            reactor.stop()
#
#        def connected_cb(conn):
#            print "Listening..."
#            conns = {"conn": conn}
#            store = ObjectStoreAsync(conns, self.args['username'], self.appid, "127.0.0.1") # TODO get the IP a better way? does it matter here?
#            store.listen(observer)
#
#        d = database.connect_box_raw(self.args['box'], self.args['username'], self.args['password'])
#        d.addCallbacks(connected_cb, err_cb)
#        reactor.run()


    def add_file(self):
        """ Add a file to the database. """
        logging.debug("Adding a file to server '{0}' in box '{1}', with id: {2}".format(self.args['server'], self.args['box'], self.args['id']))
        return self.indx.add_file(self.args['version'], self.args['id'], self.args['data'].read(), self.args['contenttype'])


    def delete_file(self):
        """ Delete a file from the database. """
        logging.debug("Deleting a file from server '{0}' in box '{1}', with id: {2}".format(self.args['server'], self.args['box'], self.args['id']))
        return self.indx.delete_file(self.args['version'], self.args['id'])


    def get_file(self):
        """ Get a file from the database. """
        logging.debug("Calling get_file on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
        # returns the file to stdout so it can be piped to a file
        print self.indx.get_file(self.args['id'])


    def list_files(self):
        """ Get a list of the files from the database. """
        logging.debug("Calling list_files on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
        return self.indx.list_files()

    def set_acl(self):
        """ Set an ACL for a target user for a database. """
        logging.debug("Calling set_acl on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
        return self.indx.set_acl(self.args['acl'], self.args['target_username'])

    def get_acls(self):
        """ Get ACLs for a database. """
        logging.debug("Calling get_acls on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
        return self.indx.get_acls()

    def generate_new_key(self):
        """ Generate and store a new key, returning the public and public-hash parts of the key. """
        logging.debug("Calling generate_new_key on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
        return self.indx.generate_new_key()

    def create_root_box(self):
        """ Create root box for a user. """
        logging.debug("Calling create_root_box on server '{0}' for box '{1}'".format(self.args['server'], self.args['box']))
        return self.indx.create_root_box(self.args['box'])

    def create_user(self):
        """ Create a new user. """
        logging.debug("Calling create_user on server '{0}' with target username '{1}'".format(self.args['server'], self.args['target_username']))
        return self.indx.create_user(self.args['target_username'], self.args['target_password'])

    def link_remote_box(self):
        """ Link a remote box with a local box. """
        logging.debug("Calling link_remote_box on remote_address '{0}', remote_box '{1}', remote_token '{2}'".format(self.args['remote_address'], self.args['remote_box'], self.args['remote_token']))
        return self.indx.link_remote_box(self.args['remote_address'], self.args['remote_box'], self.args['remote_token'])

    def generate_token(self):
        """ Generate a token for this box, and print it. """
        logging.debug("Calling generate_token on box {0}.".format(self.args['box']))
        return_d = Deferred()
        def token_cb(token):
            return_d.callback({"code": 200, "data": token})
        self.auth_and_get_token(True).addCallbacks(token_cb, return_d.errback)
        return return_d
예제 #20
0
class instagramService:
    def __init__(self, config):
        self.config = config
        self.api = InstagramAPI(access_token=config["access_token"])
        self.version = 0
        self.batch = []
        self.batch_users = []
        self.feed_count = 0
        self.feed_count_total = 0
        self.instagram_username = config["instagram_username"]
        self.instagram_user_id = config["instagram_user_id"]

    def stop_and_exit_service(self):
        logging.debug("instagram Service - HARD QUIT - instagram SERVICE")
        sys.exit()

    def run_main_services(self):

        main_services_d = Deferred()
        try:
            print "running instagram main services"

            def find_followers_cb(res):
                def find_friends_cb(res):
                    def get_authen_user_feed_cb(res):
                        def get_popular_media_cb(res):

                            main_services_d.callback(True)

                        self.get_popular_media().addCallbacks(
                            get_popular_media_cb, lambda failure: logging.error("Update Error {0}".format(failure))
                        )

                    self.get_authenticated_user_feed().addCallbacks(
                        get_authen_user_feed_cb, lambda failure: logging.error("Update Error {0}".format(failure))
                    )

                self.find_friends(self.config["instagram_user_id"]).addCallbacks(
                    find_friends_cb, lambda failure: logging.error("Update Error{0}".format(failure))
                )

            self.find_followers(self.config["instagram_user_id"]).addCallbacks(
                find_followers_cb, lambda failure: logging.error("Update Error{0}".format(failure))
            )
            # self.subscribe_to_objects_by_tag(self.config['instagram_search_words'])
        except:
            logging.debug("Service Instagram - Could not run main service due to error: {0}".format(sys.exc_info()))

        return main_services_d

    def get_indx(self):

        return_d = Deferred()

        def authed_cb():
            logging.debug("in service_tweets - Authed Callback")
            logging.debug("in service_tweets - get_indx authclient Auth status: {0}".format(authclient.is_authed))

            def token_cb(token):
                self.indx_con = IndxClient(
                    self.config["address"], self.config["box"], appid, token=token, client=authclient.client
                )
                return_d.callback(True)

            authclient.get_token(self.config["box"]).addCallbacks(token_cb, return_d.errback)

        def authed_cb_fail(re):
            logging.debug("in service tweets - get_indx/authed_cb failed for reason {0}".format(re))
            return_d.errback

        logging.debug("in service_instagram - get_indx")
        authclient = IndxClientAuth(self.config["address"], appid)
        authclient.auth_plain(self.config["user"], self.config["password"]).addCallbacks(
            lambda response: authed_cb(), authed_cb_fail
        )

        return return_d

    def get_authenticated_user_feed(self):
        auth_d = Deferred()

        def found_cb(results):
            friends_number = 0
            followers_number = 0
            since_id = ""
            # let's see if the object has some nice things in it.
            try:
                config_returned = results["data"]["service_instagram_config"]
                friends_number = int(config_returned["friends_list_size"][0]["@value"])
                followers_number = int(config_returned["followers_list_size"][0]["@value"])
                since_id = int(config_returned["since_id"][0]["@value"])
                logging.info("Found the instagram Config Object.")
            except:
                # print sys.exc_info()
                pass

            # print "Getting Auth User's feed"
            user_feed = self.api.user_media_feed()[0]
            try:
                latest_id = user_feed[0].id
            except:
                latest_id = "Null"
            ##find the highest id...

            if latest_id != since_id:

                logging.info("Found some new media, will insert it to INDX")
                since_id = latest_id
                objects_to_insert = []
                current_timestamp = str(time.time()).split(".")[0]  # str(datetime.now())
                timestamp = str(datetime.now().isoformat("T")).split(".")[0]
                current_timestamp = str(datetime.now())

                # the user responsible for this

                # now create some nice user objects...
                for media in user_feed:

                    media_user_id = media.user.id
                    media_username = media.user.username

                    uniq_id = "instagram_media_" + media.id
                    user_feed_obj = {
                        "@id": uniq_id,
                        "app_object": appid,
                        "timestamp": timestamp,
                        "type": "post",
                        "instagram_user_id_indx": "instagram_user_me",
                        "instagram_user_id": media_user_id,
                        "instagram_username": media_username,
                        "media_id": media.id,
                        "media_caption": media.caption,
                        "media_url": media.get_standard_resolution_url(),
                        "media_like_count": media.like_count,
                    }

                    # need to add INDX objects for tags and locations and the indx user of this

                    # create location if available
                    try:
                        location = media.location
                        uniq_id = "instagram_location_" + location.id
                        location_obj = {
                            "@id": uniq_id,
                            "app_object": appid,
                            "timestamp": timestamp,
                            "type": "location",
                            "instagram_location_id": location.id,
                            "location_name": location.name,
                            "latitude": location.point.latitude,
                            "longitude": location.point.longitude,
                        }

                        # now add this location to the user_feed_obj
                        user_feed_obj["media_location_id"] = location_obj
                        # thhen add it to a list of ojects to insert.
                        objects_to_insert.append(location_obj)
                    except:
                        pass

                    try:
                        tag_ids = []
                        for tag in media.tags:
                            uniq_id = "instagram_tag_" + tag.name
                            tag_obj = {
                                "@id": uniq_id,
                                "app_object": appid,
                                "timestamp": timestamp,
                                "type": "tag",
                                "instagram_tag_name": tag.name,
                            }
                            tag_ids.append(uniq_id)
                            objects_to_insert.append(tag_obj)

                        # now add this location to the user_feed_obj
                        user_feed_obj["tags"] = tag_ids
                    except:
                        pass

                    objects_to_insert.append(user_feed_obj)

                # now create the instagram_user_me object
                uniq_id = "instagram_user_me"
                instagram_me_obj = {
                    "@id": uniq_id,
                    "app_object": appid,
                    "timestamp": timestamp,
                    "type": "user",
                    "instagram_user_id": media_user_id,
                    "instagram_username": media_username,
                }
                objects_to_insert.append(instagram_me_obj)

                # and create the instagram config
                instagram_config_obj = {
                    "@id": "service_instagram_config",
                    "app_object": appid,
                    "type": "config",
                    "config_last_updated_at": timestamp,
                    "config_for_instagram_user": self.instagram_username,
                    "friends_list_generated_at": timestamp,
                    "follower_list_generated_at": timestamp,
                    "friends_list_size": friends_number,
                    "followers_list_size": followers_number,
                    "since_id": since_id,
                }
                objects_to_insert.append(instagram_config_obj)

                def update_cb(re):
                    logging.debug("network harvest async worked {0}".format(re))
                    auth_d.callback(True)

                def update_cb_fail(re):
                    logging.error("network harvest async failed {0}".format(re))
                    auth_d.errback

                self.insert_object_to_indx(objects_to_insert).addCallbacks(update_cb, update_cb_fail)
            else:
                logging.info("already have the latest version of your instagram timeline")
                auth_d.callback(True)

        def error_cb(re):
            found_cb()

        def_search = Deferred()
        find_instagram_config = {"@id": "service_instagram_config"}
        logging.info("Searching for instagram_config to check if Popular feed already harvested... ")
        def_search = self.indx_con.query(json.dumps(find_instagram_config))
        def_search.addCallbacks(found_cb, error_cb)

        return auth_d
        # for feed in user_feed:
        #     print feed

    def get_searched_media(self, search_terms):
        print "getting searched media for terms: " + str(search_terms)
        returned_results, page_num = self.api.tag_search(search_terms, 20)
        return returned_results
        # for result in returned_results:
        #     print result

    def get_popular_media(self):
        pop_d = Deferred()

        def found_cb(results):
            friends_number = 0
            followers_number = 0
            since_id = ""
            # let's see if the object has some nice things in it.
            try:
                config_returned = results["data"]["service_instagram_config"]
                friends_number = int(config_returned["friends_list_size"][0]["@value"])
                followers_number = int(config_returned["followers_list_size"][0]["@value"])
                since_id = int(config_returned["since_id"][0]["@value"])
                logging.info("Found the instagram Config Object.")
            except:
                # print sys.exc_info()
                pass

            # if(since_id > 0):
            logging.info("getting popular media")

            objects_to_insert = []
            current_timestamp = str(time.time()).split(".")[0]  # str(datetime.now())
            timestamp = str(datetime.now().isoformat("T")).split(".")[0]
            current_timestamp = str(datetime.now())

            popular_media = self.api.media_popular(count=20)

            for media in popular_media:

                media_user_id = media.user.id
                media_username = media.user.username

                # now create the instagram_user object
                uniq_user_id = "instagram_user_" + media.user.id
                instagram_media_obj = {
                    "@id": uniq_user_id,
                    "app_object": appid,
                    "timestamp": timestamp,
                    "type": "user",
                    "instagram_user_id": media_user_id,
                    "instagram_username": media_username,
                }
                objects_to_insert.append(instagram_media_obj)

                uniq_id = "instagram_media_" + media.id
                media_obj = {
                    "@id": uniq_id,
                    "app_object": appid,
                    "timestamp": timestamp,
                    "type": "post",
                    "instagram_user_id_indx": uniq_user_id,
                    "instagram_user_id": media_user_id,
                    "instagram_username": media_username,
                    "media_id": media.id,
                    "media_caption": media.caption,
                    "media_url": media.get_standard_resolution_url(),
                    "media_like_count": media.like_count,
                }

                # need to add INDX objects for tags and locations and the indx user of this

                # create location if available
                try:
                    location = media.location
                    uniq_id = "instagram_location_" + location.id
                    location_obj = {
                        "@id": uniq_id,
                        "app_object": appid,
                        "timestamp": timestamp,
                        "type": "location",
                        "instagram_location_id": location.id,
                        "location_name": location.name,
                        "latitude": location.point.latitude,
                        "longitude": location.point.longitude,
                    }

                    # now add this location to the user_feed_obj
                    media_obj["media_location_id"] = location_obj
                    # thhen add it to a list of ojects to insert.
                    objects_to_insert.append(location_obj)
                except:
                    pass

                try:
                    tag_ids = []
                    for tag in media.tags:
                        uniq_id = "instagram_tag_" + tag.name
                        tag_obj = {
                            "@id": uniq_id,
                            "app_object": appid,
                            "type": "tag",
                            "timestamp": timestamp,
                            "instagram_tag_name": tag.name,
                        }
                        tag_ids.append(uniq_id)
                        objects_to_insert.append(tag_obj)

                    # now add this location to the user_feed_obj
                    media_obj["tags"] = tag_ids
                except:
                    pass

                objects_to_insert.append(media_obj)

            def update_cb(re):
                logging.debug("network harvest async worked {0}".format(re))
                pop_d.callback(True)

            def update_cb_fail(re):
                logging.error("network harvest async failed {0}".format(re))
                pop_d.errback

            # and create the instagram config
            instagram_config_obj = {
                "@id": "service_instagram_config",
                "app_object": appid,
                "type": "config",
                "config_last_updated_at": timestamp,
                "config_for_instagram_user": self.instagram_username,
                "friends_list_generated_at": timestamp,
                "follower_list_generated_at": timestamp,
                "friends_list_size": friends_number,
                "followers_list_size": followers_number,
                "since_id": since_id,
            }
            objects_to_insert.append(instagram_config_obj)

            self.insert_object_to_indx(objects_to_insert).addCallbacks(update_cb, update_cb_fail)

        def error_cb(re):
            found_cb()

        def_search = Deferred()
        find_instagram_config = {"@id": "service_instagram_config"}
        logging.info("Searching for instagram_config to check if Popular feed already harvested... ")
        def_search = self.indx_con.query(json.dumps(find_instagram_config))
        def_search.addCallbacks(found_cb, error_cb)

        return pop_d

    def find_user(self, username):
        data = self.api.user_search(username, count=20)
        print data

    def find_followers(self, userid):

        # first do a lookup and see if the latest set if followers has allready been found
        follower_d = Deferred()

        def found_cb(results):
            friends_number = 0
            followers_number = 0
            since_id = 0
            # let's see if the object has some nice things in it.
            try:
                config_returned = results["data"]["service_instagram_config"]
                friends_number = int(config_returned["friends_list_size"][0]["@value"])
                followers_number = int(config_returned["followers_list_size"][0]["@value"])
                since_id = int(config_returned["since_id"][0]["@value"])
                logging.info("Found the instagram Config Object.")
            except:
                # print sys.exc_info()
                pass
            followed_by = self.api.user_followed_by(userid)[0]
            # print "followed_by length: "+str(len(followed_by))
            # print str(followed_by)
            # see if the number is different, if it is, then update.. could be nicer than this, but for now, it will work (ish)
            if (len(followed_by) != followers_number) or (followers_number == 0):

                followers_number = len(followed_by)
                objects_to_insert = []
                current_timestamp = str(time.time()).split(".")[0]  # str(datetime.now())
                timestamp = str(datetime.now().isoformat("T")).split(".")[0]
                uniq_id = "instagram_follower_network_for_user_me"  # +self.instagram_username
                followed_by_obj = {
                    "@id": uniq_id,
                    "app_object": appid,
                    "instagram_username": self.instagram_username,
                    "instagram_user_id": self.instagram_user_id,
                    "timestamp": timestamp,
                    "followed_by_count": len(followed_by),
                }

                followers_ids = []
                for follower in followed_by:
                    # print follower.username
                    uniq_id = "instagram_user_" + str(follower.username)
                    follower_obj = {
                        "@id": uniq_id,
                        "app_object": appid,
                        "type": "user",
                        "instagram_username": str(follower.username),
                        "timestamp": timestamp,
                    }
                    objects_to_insert.append(follower_obj)
                    # we can add this to the followed_by_obj later
                    followers_ids.append(uniq_id)

                # link the followers for me
                followed_by_obj["follower_ids"] = followers_ids
                # print followed_by_objs
                # for friend in friends_list:
                # print friend
                # now append the results
                def update_cb(re):
                    logging.debug("network harvest async worked {0}".format(re))
                    follower_d.callback(True)

                def update_cb_fail(re):
                    logging.error("network harvest async failed {0}".format(re))
                    follower_d.errback

                instagram_config_obj = {
                    "@id": "service_instagram_config",
                    "app_object": appid,
                    "type": "config",
                    "config_last_updated_at": timestamp,
                    "config_for_instagram_user": self.instagram_username,
                    "friends_list_generated_at": timestamp,
                    "follower_list_generated_at": timestamp,
                    "friends_list_size": friends_number,
                    "followers_list_size": followers_number,
                    "since_id": since_id,
                }

                objects_to_insert.append(followed_by_obj)
                # objects_to_insert.append(followers)
                objects_to_insert.append(instagram_config_obj)

                self.insert_object_to_indx(objects_to_insert).addCallbacks(update_cb, update_cb_fail)
            else:
                follower_d.callback(True)

        def error_cb(re):
            found_cb()

        def_search = Deferred()
        find_instagram_config = {"@id": "service_instagram_config"}
        logging.info("Searching for instagram_config to check if network already harvested... ")
        def_search = self.indx_con.query(json.dumps(find_instagram_config))
        def_search.addCallbacks(found_cb, error_cb)

        return follower_d
        # print followed_by

    def find_friends(self, userid):
        friends_d = Deferred()

        def found_cb(results):
            friends_number = 0
            followers_number = 0
            since_id = 0
            # let's see if the object has some nice things in it.
            try:
                config_returned = results["data"]["service_instagram_config"]
                friends_number = int(config_returned["friends_list_size"][0]["@value"])
                followers_number = int(config_returned["followers_list_size"][0]["@value"])
                since_id = int(config_returned["since_id"][0]["@value"])
                logging.info("Found the instagram Config Object.")
            except:
                # print sys.exc_info()
                pass

            friends_by_list = self.api.user_follows(userid)[0]

            if (len(friends_by_list) != friends_number) or (friends_number == 0):

                friends_number = len(friends_by_list)
                objects_to_insert = []
                current_timestamp = str(time.time()).split(".")[0]  # str(datetime.now())
                timestamp = str(datetime.now().isoformat("T")).split(".")[0]
                uniq_id = "instagram_friends_network_for_user_me"  # +self.instagram_username
                friends_by_obj = {
                    "@id": uniq_id,
                    "app_object": appid,
                    "instagram_username": self.instagram_username,
                    "instagram_user_id": self.instagram_user_id,
                    "timestamp": timestamp,
                    "followed_by_count": len(friends_by_list),
                }

                friends_ids = []
                for friend in friends_by_list:
                    # print follower.username
                    uniq_id = "instagram_user_" + str(friend.username)
                    friend_obj = {
                        "@id": uniq_id,
                        "app_object": appid,
                        "type": "user",
                        "instagram_username": str(friend.username),
                        "timestamp": timestamp,
                    }
                    objects_to_insert.append(friend_obj)
                    # we can add this to the followed_by_obj later
                    friends_ids.append(uniq_id)

                friends_by_obj["friends_ids"] = friends_ids
                # print friends_by_objs
                # for friend in friends_list:
                # print friend
                # now append the results
                def update_cb(re):
                    logging.debug("network harvest async worked {0}".format(re))
                    friends_d.callback(True)

                def update_cb_fail(re):
                    logging.error("network harvest async failed {0}".format(re))
                    friends_d.errback

                instagram_config_obj = {
                    "@id": "service_instagram_config",
                    "app_object": appid,
                    "type": "config",
                    "config_last_updated_at": timestamp,
                    "config_for_instagram_user": self.instagram_username,
                    "friends_list_generated_at": timestamp,
                    "follower_list_generated_at": timestamp,
                    "friends_list_size": friends_number,
                    "followers_list_size": followers_number,
                    "since_id": since_id,
                }

                objects_to_insert.append(friends_by_obj)
                # objects_to_insert.append(followers)
                objects_to_insert.append(instagram_config_obj)

                self.insert_object_to_indx(objects_to_insert).addCallbacks(update_cb, update_cb_fail)
            else:
                friends_d.callback(True)

        def error_cb(re):
            found_cb()

        def_search = Deferred()
        find_instagram_config = {"@id": "service_instagram_config"}
        logging.info("Searching for instagram_config to check if network already harvested... ")
        def_search = self.indx_con.query(json.dumps(find_instagram_config))
        def_search.addCallbacks(found_cb, error_cb)

        return friends_d
        # print followed_by

    def subscribe_to_objects_by_tag(self, search_terms):
        def process_tag_update(update):
            print update

        reactor = subscriptions.SubscriptionsReactor()
        reactor.register_callback(subscriptions.SubscriptionType.TAG, process_tag_update)
        self.api.create_subscription(
            object="tag", object_id=search_terms, aspect="media", callback_url="http://*****:*****@version"]
            logging.debug(
                "Succesfully Updated INDX with Objects in update_cb, new diff version of {0}".format(self.version)
            )
            # logging.debug("Inserted Object into INDX: ".format(resp))
            update_d.callback(True)
            # return update_d
            # time.sleep(2)

        def exception_cb(e, service=self, obj=obj):
            logging.debug("Exception Inserting into INDX, probably wrong version given")
            if isinstance(e.value, urllib2.HTTPError):  # handle a version incorrect error, and update the version
                if e.value.code == 409:  # 409 Obsolete
                    response = e.value.read()
                    json_response = json.loads(response)
                    service.version = json_response["@version"]
                    logging.debug("INDX insert error in Instagram Service Object: " + str(response))
                    try:
                        service.indx_con.update(service.version, obj).addCallbacks(update_cb, exception_cb)
                        # logging.debug('Twitter Service - Successfully added Objects into Box')
                    except:
                        logging.error("Instagram Service, error on insert {0}".format(response))
                        update_d.errback(e.value)
                else:
                    logging.error("Instagram Service Unknow error: {0}".format(e.value.read()))
                    update_d.errback(e.value)
            else:
                logging.error("Error updating INDX: {0}".format(e.value))
                update_d.errback(e.value)

        logging.debug(
            "in Instagram - Trying to insert into indx...Current diff version: {0} and objects (len) given {1}".format(
                self.version, len(obj)
            )
        )
        self.indx_con.update(self.version, obj).addCallbacks(update_cb, exception_cb)

        return update_d