def token_cb(token):
self.indx_con = IndxClient(self.config['address'],
self.config['box'],
appid,
token=token,
client=authclient.client)
return_d.callback(True)
def token_cb(token):
if not self.indx:
self.indx = IndxClient(self.args['server'],
self.args['box'],
self.appid,
token=token,
client=self.client)
do_call()
def call_action(self, name, *args, **kwargs):
""" Calls an action by name. """
action = self.actions[name]
f = action['f']
self.check_args(action['args'])
if not self.indx:
self.indx = IndxClient(self.args['server'], self.args['box'],
self.args['username'],
self.args['password'], self.appid)
return self.parse_status(name, f(*args, **kwargs))
def call_action(self, name, *args, **kwargs):
""" Calls an action by name. """
return_d = Deferred()
action = self.actions[name]
f = action['f']
self.check_args(action['args'])
def do_call():
f(*args, **kwargs).addCallbacks(
lambda status: return_d.callback(
self.parse_status(name, status)), return_d.errback)
if not self.token:
def token_cb(token):
if not self.indx:
self.indx = IndxClient(self.args['server'],
self.args['box'],
self.appid,
token=token,
client=self.client)
do_call()
self.auth_and_get_token(IndxClient.requires_token(f)).addCallbacks(
token_cb, return_d.errback)
else:
do_call()
return return_d
def token_cb(token):
indx = IndxClient(server_url,
self.config_box,
"INDX_Fitbit_Harvester",
token=token,
client=authclient.client)
indx_d.callback(indx)
def token_cb(token):
indx = IndxClient(server_url,
box,
"Slicer",
token=token,
client=authclient.client)
logging.debug("Got the indx client token")
indx_d.callback(indx)
def call_action(self, name, *args, **kwargs):
""" Calls an action by name. """
action = self.actions[name]
f = action['f']
self.check_args(action['args'])
if not self.indx:
self.indx = IndxClient(self.args['server'], self.args['box'], self.args['username'], self.args['password'], self.appid)
return self.parse_status(name, f(*args, **kwargs))
def token_cb(remote_token):
logging.debug("IndxSync sync_boxes token_cb")
client = IndxClient(remote_server_url, remote_box, self.APPID, client = clientauth.client, token = remote_token, keystore = self.keystore)
def updated_cb(empty):
def pushed_cb(empty):
def observer(data):
if data.get('action') == 'diff' and data.get('operation') == 'update':
diff = data['data']
def done_cb(empty):
logging.debug("IndxSync updating from a websocket done.")
def err_cb(failure):
logging.error("IndxSync updating from a websocket error: {0}".format(failure))
self.update_to_latest_version(client, remote_server_url, remote_box, diff_in = diff).addCallbacks(done_cb, err_cb)
else:
logging.error("Sync: Unknown data message from WebSocket: {0}".format(data))
# auths and sets up listening for diffs, filtering them and passing them to the observer
if websocket is None:
wsclient = client.connect_ws(local_key['key']['private'], local_key_hash, observer, remote_encpk2, self.webserver) # open a new socket
else:
websocket.listen_remote(local_key['key']['private'], local_key_hash, observer, remote_encpk2) # use an existing websocket
next_model(None)
if not include_push_all:
pushed_cb(None)
return
# push the whole box to the remote box
def latest_cb(graph):
def ver_cb(version):
client.update_raw(version, graph.to_flat_json()).addCallbacks(pushed_cb, return_d.errback)
client.get_version().addCallbacks(lambda resp: ver_cb(resp['data']), return_d.errback)
self.root_store.get_latest(render_json = False).addCallbacks(latest_cb, return_d.errback)
# compare local version to previous, and update one of them, or both
self.update_to_latest_version(client, remote_server_url, remote_box).addCallbacks(updated_cb, return_d.errback)
def call_action(self, name, *args, **kwargs):
""" Calls an action by name. """
return_d = Deferred()
action = self.actions[name]
f = action['f']
self.check_args(action['args'])
def do_call():
f(*args, **kwargs).addCallbacks(lambda status: return_d.callback(self.parse_status(name, status)), return_d.errback)
if not self.token:
def token_cb(token):
if not self.indx:
self.indx = IndxClient(self.args['server'], self.args['box'], self.appid, token = token, client = self.client)
do_call()
self.auth_and_get_token(IndxClient.requires_token(f)).addCallbacks(token_cb, return_d.errback)
else:
do_call()
return return_d
def ver_cb(ver):
def new_remote_key_cb(remote_keys):
logging.debug("IndxSync link_remote_box, new_remote_key_cb, remote_keys: {0}".format(remote_keys))
# NB: no "private" in remote_keys, that never leaves the remote box.
remote_keys = remote_keys['data'] # remove the Indx HTTP response padding
remote_encpk = remote_keys['encpk2']
if type(remote_encpk) != type(""):
remote_encpk = json.dumps(remote_encpk)
remote_serverid = remote_keys['serverid']
link_uid = uuid.uuid1()
local_key_uid = uuid.uuid1()
remote_key_uid = uuid.uuid1()
status1_uid = uuid.uuid1()
status2_uid = uuid.uuid1()
link_uri = "link-{0}".format(link_uid)
status1_uri = "status-{0}".format(status1_uid)
status2_uri = "status-{0}".format(status2_uid)
# objects get updated to local box, and then synced across to the other boxes once the syncing starts
new_objs = [
{ "@id": link_uri,
"type": [ {"@value": NS_ROOT_BOX + "link"} ],
"boxes": [ {"@id": "box-{0}".format(local_key_uid)}, # links to the objs below
{"@id": "box-{0}".format(remote_key_uid)}, # links to the objs below
],
"statuses": [ {"@id": status1_uri}, {"@id": status2_uri} ],
},
{ "@id": status1_uri,
"type": [ {"@value": NS_ROOT_BOX + "status"} ],
"src-boxid": [ {"@value": remote_box} ],
"src-server-url": [ {"@value": remote_address} ],
"dst-boxid": [ {"@value": self.root_store.boxid} ],
"dst-server-url": [ {"@value": self.url} ],
"last-version-seen": [ {"@value": ver + 1} ], #we save this because we will push our whole box to it in a sec - it is incremented because the version is before we put this in
},
{ "@id": status2_uri,
"type": [ {"@value": NS_ROOT_BOX + "status"} ],
"dst-boxid": [ {"@value": remote_box} ],
"dst-server-url": [ {"@value": remote_address} ],
"src-boxid": [ {"@value": self.root_store.boxid} ],
"src-server-url": [ {"@value": self.url} ],
},
{ "@id": "box-{0}".format(local_key_uid),
"type": [ {"@value": NS_ROOT_BOX + "box"} ],
"server-url": [ {"@value": self.url } ],
"server-id": [ {"@value": server_id } ],
"box": [ {"@value": self.root_store.boxid} ],
"key": [ {"@id": local_keys['public-hash']}], # links to the key objs below
},
{ "@id": "box-{0}".format(remote_key_uid),
"type": [ {"@value": NS_ROOT_BOX + "box"} ],
"server-url": [ {"@value": remote_address } ],
"server-id": [ {"@value": remote_serverid } ],
"box": [ {"@value": remote_box} ],
"key": [ {"@id": remote_keys['public-hash']}], # links to the key objs below
},
{ "@id": local_keys['public-hash'],
"type": [ {"@value": NS_ROOT_BOX + "key"} ],
"public-key": [ {"@value": local_keys['public']} ], # share the full public keys everywhere (private keys only in respective server's keystores)
"public-hash": [ {"@value": local_keys['public-hash']} ], # share the full public keys everywhere (private keys only in respective server's keystores)
},
{ "@id": remote_keys['public-hash'],
"type": [ {"@value": NS_ROOT_BOX + "key"} ],
"public-key": [ {"@value": remote_keys['public']} ], # share the full public keys everywhere
"public-hash": [ {"@value": remote_keys['public-hash']} ], # share the full public keys everywhere
},
]
def update_cb(empty):
def diff_cb(diff):
def applied_cb(empty):
def encpk_cb(empty):
def remote_added_cb(empty):
def local_added_cb(empty):
logging.debug("IndxSync link_remote_box, local_added_cb")
def added_indx_cb(empty):
# start syncing/connecting using the new key
self.sync_boxes([link_uri], include_push_all = True).addCallbacks(lambda empty: return_d.callback(remote_keys['public']), return_d.errback)
self.database.save_linked_box(self.root_store.boxid).addCallbacks(added_indx_cb, return_d.errback)
# add the local key to the local store
self.keystore.put(local_keys, local_user, self.root_store.boxid).addCallbacks(local_added_cb, return_d.errback) # store in the local keystore
self.keystore.put({"public": remote_keys['public'], "private": "", "public-hash": remote_keys['public-hash']}, local_user, self.root_store.boxid).addCallbacks(remote_added_cb, return_d.errback)
# don't save the local encpk2 here, only give it to the remote server.
# save the remote encpk2
self.database.save_encpk2(sha512_hash(remote_encpk), remote_encpk, remote_serverid).addCallbacks(encpk_cb, return_d.errback)
client.apply_diff(diff).addCallbacks(applied_cb, return_d.errback)
self.root_store.diff(0, None, "diff").addCallbacks(diff_cb, return_d.errback)
self.root_store.update(new_objs, ver, propagate = True).addCallbacks(update_cb, return_d.errback)
client = IndxClient(remote_address, remote_box, self.APPID, token = remote_token)
client.generate_new_key(local_keys, local_encpk, server_id).addCallbacks(new_remote_key_cb, return_d.errback)
class CLIClient:
def __init__(self, appid="INDX CLIClient"):
""" Associate command-line actions with functions, and enforce required variables. """
self.actions = {
'create_box': {
'f': self.create_box,
'args': ['box']
},
'delete_box': {
'f': self.delete_box,
'args': ['box']
},
'list_boxes': {
'f': self.list_boxes,
'args': []
},
'get_object_ids': {
'f': self.get_object_ids,
'args': ['box']
},
'create_user': {
'f': self.create_user,
'args': ['target_username', 'target_password']
},
'update': {
'f': self.update,
'args': ['box', 'data', 'version']
},
'update_json': {
'f': self.update_json,
'args': ['box', 'data', 'version']
},
'update_raw': {
'f': self.update_raw,
'args': ['box', 'data', 'version']
},
'delete': {
'f': self.delete,
'args': ['box', 'id', 'version']
},
'get_latest': {
'f': self.get_latest,
'args': ['box']
},
'get_by_ids': {
'f': self.get_by_ids,
'args': ['box', 'id']
},
'query': {
'f': self.query,
'args': ['box', 'query']
},
'diff': {
'f': self.diff,
'args': ['box', 'from', 'return_objs']
},
#'listen': {'f': self.listen, 'args': ['box']},
'add_file': {
'f': self.add_file,
'args': ['box', 'data', 'id', 'version', 'contenttype']
},
'delete_file': {
'f': self.delete_file,
'args': ['box', 'id', 'version']
},
'get_file': {
'f': self.get_file,
'args': ['box', 'id']
},
'list_files': {
'f': self.list_files,
'args': ['box']
},
'set_acl': {
'f': self.set_acl,
'args': ['box', 'acl', 'target_username']
},
'set_acl_public': {
'f': self.set_acl_public,
'args': ['box', 'acl']
},
'get_acls': {
'f': self.get_acls,
'args': ['box']
},
'generate_new_key': {
'f': self.generate_new_key,
'args': ['box']
},
'create_root_box': {
'f': self.create_root_box,
'args': ['box']
},
'link_remote_box': {
'f': self.link_remote_box,
'args':
['box', 'remote_token', 'remote_box', 'remote_address']
},
'generate_token': {
'f': self.generate_token,
'args': ['box']
},
}
self.token = None
self.appid = appid
self.indx = None
def set_args(self, args):
""" Move relevant command-line arguments into local variable. """
logging.debug("Set args: {0}".format(args))
self.args = args
""" Flatten single value lists into flat key/value pair. """
for key in args:
if type(args[key]) == type([]) and len(args[key]) == 1:
args[key] = args[key][0]
""" Ensure self.server always ends in a / """
if self.args['server'][-1:] != "/":
self.args['server'] += "/"
def check_args(self, required):
not_set = []
for key in required:
if key not in self.args or self.args[key] is None or self.args[
key] == "":
not_set.append(key)
if not self.args['allowempty'] and len(not_set) > 0:
raise Exception(
"The following values cannot be empty for this action: {0}".
format(", ".join(not_set)))
def auth_and_get_token(self, get_token):
""" Authenticate, get a token and call it back to the deferred. """
return_d = Deferred()
def authed_cb():
def token_cb(token):
if token is not None:
self.token = token
return_d.callback(token)
else:
return_d.callback(None)
if get_token:
authclient.get_token(self.args['box']).addCallbacks(
token_cb, return_d.errback)
else:
token_cb(None)
authclient = IndxClientAuth(self.args['server'], self.appid)
self.client = authclient.client
authclient.auth_plain(self.args['username'],
self.args['password']).addCallbacks(
lambda response: authed_cb(),
return_d.errback)
return return_d
def call_action(self, name, *args, **kwargs):
""" Calls an action by name. """
return_d = Deferred()
action = self.actions[name]
f = action['f']
self.check_args(action['args'])
def do_call():
f(*args, **kwargs).addCallbacks(
lambda status: return_d.callback(
self.parse_status(name, status)), return_d.errback)
if not self.token:
def token_cb(token):
if not self.indx:
self.indx = IndxClient(self.args['server'],
self.args['box'],
self.appid,
token=token,
client=self.client)
do_call()
self.auth_and_get_token(IndxClient.requires_token(f)).addCallbacks(
token_cb, return_d.errback)
else:
do_call()
return return_d
def parse_status(self, source, status):
""" Parse the status returned from the server, and raise an Exception if necessary. """
if status is not None: # status is None when a file has been printed raw
if status['code'] < 200 or status['code'] > 299:
raise Exception(
"{0} in box {1} failed. Response is {2} with code {3}".
format(source, self.args['box'], status['message'],
status['code']))
else:
if "data" in status and self.args['jsondata']:
return json.dumps(status['data'], indent=2)
else:
pretty = pprint.pformat(status, indent=2, width=80)
logging.info(
"{0} in box {1} successful, return is: {2}".format(
source, self.args['box'], pretty))
return None
""" Test functions."""
def create_box(self):
""" Test to create a box. """
logging.debug("Creating box: '{0}' on server '{1}'".format(
self.args['box'], self.args['server']))
return self.indx.create_box()
def delete_box(self):
""" Test to delete a box. """
logging.debug("Deleting box: '{0}' on server '{1}'".format(
self.args['box'], self.args['server']))
return self.indx.delete_box()
def list_boxes(self):
""" List the boxes on the INDX server. """
logging.debug("Listing boxes on server '{0}'".format(
self.args['server']))
return self.indx.list_boxes()
def get_object_ids(self):
""" Get the IDs of every object in this box. """
logging.debug(
"Getting a list of object IDs on server '{0}' in box '{1}'".format(
self.args['server'], self.args['box']))
return self.indx.get_object_ids()
def update(self):
""" Test to update objects in a box. """
logging.debug("Updating data to box: '{0}' on server '{1}'".format(
self.args['box'], self.args['server']))
#return self.indx.update(self.args['version'], cjson.decode(self.args['data'].read()), all_unicode=True)
return self.indx.update(self.args['version'],
json.loads(self.args['data'].read()))
def update_json(self):
""" Test to update json objects in a box. """
logging.debug("Updating data to box: '{0}' on server '{1}'".format(
self.args['box'], self.args['server']))
#return self.indx.update(self.args['version'], cjson.decode(self.args['data'].read()), all_unicode=True)
return self.indx.update_json(self.args['version'],
json.loads(self.args['data'].read()))
def update_raw(self):
""" Test to update raw objects in a box. """
logging.debug("Updating raw data to box: '{0}' on server '{1}'".format(
self.args['box'], self.args['server']))
#return self.indx.update(self.args['version'], cjson.decode(self.args['data'].read()), all_unicode=True)
return self.indx.update_raw(self.args['version'],
json.loads(self.args['data'].read()))
def delete(self):
""" Test to delete objects from a box.
'id' argument should be a list of ids
e.g., --id=id1 --id=id2
"""
logging.debug("Deleting data to box: '{0}' on server '{1}'".format(
self.args['box'], self.args['server']))
return self.indx.delete(self.args['version'], self.args['id'])
def get_latest(self):
""" Get the latest version of every object in this box. """
logging.debug(
"Getting latest objects on server '{0}' in box '{1}'".format(
self.args['server'], self.args['box']))
return self.indx.get_latest()
def get_by_ids(self):
""" Get the latest version of specific objects in this box. """
logging.debug(
"Getting latest objects on server '{0}' in box '{1}'".format(
self.args['server'], self.args['box']))
ids = self.args['id']
if type(ids) != type([]):
ids = [ids] # put a single id into an array
return self.indx.get_by_ids(ids)
def query(self):
""" Query this box.
e.g., query="{ '@id': 2983 }"
or query="{ 'firstname': 'dan' }"
"""
logging.debug(
"Querying server '{0}' in box '{1}' with depth '{2}'".format(
self.args['server'], self.args['box'], self.args['depth']))
return self.indx.query(self.args['query'],
depth=int(self.args['depth']))
def diff(self):
""" Run a diff of two version of this box. """
logging.debug("Calling diff on server '{0}' in box '{1}'".format(
self.args['server'], self.args['box']))
to_version = None
if "to" in self.args and self.args['to'] is not None:
to_version = self.args['to']
return self.indx.diff(self.args['return_objs'],
self.args['from'],
to_version=to_version)
# def listen(self):
# """ Listen to updates to the database (locally, not with HTTP) and print out the diff in realtime. """
# self.check_args(['box', 'username', 'password'])
#
# def observer(notify):
# print "Version updated to: {0}".format(notify.payload)
#
# def err_cb(failure):
# logging.error("Error in test listen: {0}".format(failure))
# reactor.stop()
#
# def connected_cb(conn):
# print "Listening..."
# conns = {"conn": conn}
# store = ObjectStoreAsync(conns, self.args['username'], self.appid, "127.0.0.1") # TODO get the IP a better way? does it matter here?
# store.listen(observer)
#
# d = database.connect_box_raw(self.args['box'], self.args['username'], self.args['password'])
# d.addCallbacks(connected_cb, err_cb)
# reactor.run()
def add_file(self):
""" Add a file to the database. """
logging.debug(
"Adding a file to server '{0}' in box '{1}', with id: {2}".format(
self.args['server'], self.args['box'], self.args['id']))
return self.indx.add_file(self.args['version'], self.args['id'],
self.args['data'].read(),
self.args['contenttype'])
def delete_file(self):
""" Delete a file from the database. """
logging.debug(
"Deleting a file from server '{0}' in box '{1}', with id: {2}".
format(self.args['server'], self.args['box'], self.args['id']))
return self.indx.delete_file(self.args['version'], self.args['id'])
def get_file(self):
""" Get a file from the database. """
logging.debug("Calling get_file on server '{0}' in box '{1}'".format(
self.args['server'], self.args['box']))
# returns the file to stdout so it can be piped to a file
print self.indx.get_file(self.args['id'])
def list_files(self):
""" Get a list of the files from the database. """
logging.debug("Calling list_files on server '{0}' in box '{1}'".format(
self.args['server'], self.args['box']))
return self.indx.list_files()
def set_acl(self):
""" Set an ACL for a target user for a database. """
logging.debug("Calling set_acl on server '{0}' in box '{1}'".format(
self.args['server'], self.args['box']))
return self.indx.set_acl(self.args['acl'],
self.args['target_username'])
def set_acl_public(self):
""" Set an ACL for the public uer for a database. """
logging.debug(
"Calling set_acl_public on server '{0}' in box '{1}'".format(
self.args['server'], self.args['box']))
return self.indx.set_acl_public(self.args['acl'])
def get_acls(self):
""" Get ACLs for a database. """
logging.debug("Calling get_acls on server '{0}' in box '{1}'".format(
self.args['server'], self.args['box']))
return self.indx.get_acls()
def generate_new_key(self):
""" Generate and store a new key, returning the public and public-hash parts of the key. """
logging.debug(
"Calling generate_new_key on server '{0}' in box '{1}'".format(
self.args['server'], self.args['box']))
return self.indx.generate_new_key()
def create_root_box(self):
""" Create root box for a user. """
logging.debug(
"Calling create_root_box on server '{0}' for box '{1}'".format(
self.args['server'], self.args['box']))
return self.indx.create_root_box(self.args['box'])
def create_user(self):
""" Create a new user. """
logging.debug(
"Calling create_user on server '{0}' with target username '{1}'".
format(self.args['server'], self.args['target_username']))
return self.indx.create_user(self.args['target_username'],
self.args['target_password'])
def link_remote_box(self):
""" Link a remote box with a local box. """
logging.debug(
"Calling link_remote_box on remote_address '{0}', remote_box '{1}', remote_token '{2}'"
.format(self.args['remote_address'], self.args['remote_box'],
self.args['remote_token']))
return self.indx.link_remote_box(self.args['remote_address'],
self.args['remote_box'],
self.args['remote_token'])
def generate_token(self):
""" Generate a token for this box, and print it. """
logging.debug("Calling generate_token on box {0}.".format(
self.args['box']))
return_d = Deferred()
def token_cb(token):
return_d.callback({"code": 200, "data": token})
self.auth_and_get_token(True).addCallbacks(token_cb, return_d.errback)
return return_d
def token_cb(token):
if not self.indx:
self.indx = IndxClient(self.args['server'], self.args['box'], self.appid, token = token, client = self.client)
do_call()
def token_cb(token):
indx = IndxClient(server_url, box, appid, token = token)
return_d.callback(indx)
help="Earliest date to download in format: 2013-01-20")
parser.add_argument('--debug',
default=False,
action="store_true",
help="Enable debugging")
args = vars(parser.parse_args())
password = getpass.getpass("Nike+ password: "******"INDX password: "******""" Assert data into the INDX.
If the version is incorrect, the correct version will be grabbed and the update re-sent.
data -- An object to assert into the box.
"""
global version
try:
response = indx.update(version, data)
version = response['data']['@version'] # update the version
except Exception as e:
if isinstance(
class instagramService:
def __init__(self, config):
self.config = config
self.api = InstagramAPI(access_token=config['access_token'])
self.version = 0
self.batch = []
self.batch_users = []
self.feed_count = 0
self.feed_count_total = 0
self.instagram_username = config['instagram_username']
self.instagram_user_id = config['instagram_user_id']
def stop_and_exit_service(self):
logging.debug('instagram Service - HARD QUIT - instagram SERVICE')
sys.exit()
def run_main_services(self):
main_services_d = Deferred()
try:
print "running instagram main services"
def find_followers_cb(res):
def find_friends_cb(res):
def get_authen_user_feed_cb(res):
def get_popular_media_cb(res):
main_services_d.callback(True)
self.get_popular_media().addCallbacks(
get_popular_media_cb, lambda failure: logging.
error("Update Error {0}".format(failure)))
self.get_authenticated_user_feed().addCallbacks(
get_authen_user_feed_cb, lambda failure: logging.error(
"Update Error {0}".format(failure)))
self.find_friends(
self.config['instagram_user_id']).addCallbacks(
find_friends_cb, lambda failure: logging.error(
"Update Error{0}".format(failure)))
self.find_followers(self.config['instagram_user_id']).addCallbacks(
find_followers_cb, lambda failure: logging.error(
"Update Error{0}".format(failure)))
#self.subscribe_to_objects_by_tag(self.config['instagram_search_words'])
except:
logging.debug(
'Service Instagram - Could not run main service due to error: {0}'
.format(sys.exc_info()))
return main_services_d
def get_indx(self):
return_d = Deferred()
def authed_cb():
logging.debug("in service_tweets - Authed Callback")
logging.debug(
"in service_tweets - get_indx authclient Auth status: {0}".
format(authclient.is_authed))
def token_cb(token):
self.indx_con = IndxClient(self.config['address'],
self.config['box'],
appid,
token=token,
client=authclient.client)
return_d.callback(True)
authclient.get_token(self.config['box']).addCallbacks(
token_cb, return_d.errback)
def authed_cb_fail(re):
logging.debug(
"in service tweets - get_indx/authed_cb failed for reason {0}".
format(re))
return_d.errback
logging.debug("in service_instagram - get_indx")
authclient = IndxClientAuth(self.config['address'], appid)
authclient.auth_plain(self.config['user'],
self.config['password']).addCallbacks(
lambda response: authed_cb(), authed_cb_fail)
return return_d
def get_authenticated_user_feed(self):
auth_d = Deferred()
def found_cb(results):
friends_number = 0
followers_number = 0
since_id = ""
#let's see if the object has some nice things in it.
try:
config_returned = results['data']['service_instagram_config']
friends_number = int(
config_returned['friends_list_size'][0]['@value'])
followers_number = int(
config_returned['followers_list_size'][0]['@value'])
since_id = int(config_returned['since_id'][0]['@value'])
logging.info('Found the instagram Config Object.')
except:
#print sys.exc_info()
pass
#print "Getting Auth User's feed"
user_feed = self.api.user_media_feed()[0]
try:
latest_id = user_feed[0].id
except:
latest_id = "Null"
##find the highest id...
if (latest_id != since_id):
logging.info("Found some new media, will insert it to INDX")
since_id = latest_id
objects_to_insert = []
current_timestamp = str(
time.time()).split(".")[0] #str(datetime.now())
timestamp = str(datetime.now().isoformat('T')).split(".")[0]
current_timestamp = str(datetime.now())
#the user responsible for this
#now create some nice user objects...
for media in user_feed:
media_user_id = media.user.id
media_username = media.user.username
uniq_id = "instagram_media_" + media.id
user_feed_obj = {
"@id": uniq_id,
"app_object": appid,
"timestamp": timestamp,
"type": "post",
"instagram_user_id_indx": "instagram_user_me",
"instagram_user_id": media_user_id,
"instagram_username": media_username,
"media_id": media.id,
"media_caption": media.caption,
"media_url": media.get_standard_resolution_url(),
"media_like_count": media.like_count
}
#need to add INDX objects for tags and locations and the indx user of this
#create location if available
try:
location = media.location
uniq_id = "instagram_location_" + location.id
location_obj = {
"@id": uniq_id,
"app_object": appid,
"timestamp": timestamp,
"type": "location",
"instagram_location_id": location.id,
"location_name": location.name,
"latitude": location.point.latitude,
"longitude": location.point.longitude
}
#now add this location to the user_feed_obj
user_feed_obj['media_location_id'] = location_obj
#thhen add it to a list of ojects to insert.
objects_to_insert.append(location_obj)
except:
pass
try:
tag_ids = []
for tag in media.tags:
uniq_id = "instagram_tag_" + tag.name
tag_obj = {
"@id": uniq_id,
"app_object": appid,
"timestamp": timestamp,
"type": "tag",
"instagram_tag_name": tag.name
}
tag_ids.append(uniq_id)
objects_to_insert.append(tag_obj)
#now add this location to the user_feed_obj
user_feed_obj['tags'] = tag_ids
except:
pass
objects_to_insert.append(user_feed_obj)
#now create the instagram_user_me object
uniq_id = "instagram_user_me"
instagram_me_obj = {
"@id": uniq_id,
"app_object": appid,
"timestamp": timestamp,
"type": "user",
"instagram_user_id": media_user_id,
"instagram_username": media_username
}
objects_to_insert.append(instagram_me_obj)
#and create the instagram config
instagram_config_obj = {
"@id": "service_instagram_config",
"app_object": appid,
"type": "config",
"config_last_updated_at": timestamp,
"config_for_instagram_user": self.instagram_username,
"friends_list_generated_at": timestamp,
"follower_list_generated_at": timestamp,
"friends_list_size": friends_number,
"followers_list_size": followers_number,
"since_id": since_id
}
objects_to_insert.append(instagram_config_obj)
def update_cb(re):
logging.debug(
"network harvest async worked {0}".format(re))
auth_d.callback(True)
def update_cb_fail(re):
logging.error(
"network harvest async failed {0}".format(re))
auth_d.errback
self.insert_object_to_indx(objects_to_insert).addCallbacks(
update_cb, update_cb_fail)
else:
logging.info(
"already have the latest version of your instagram timeline"
)
auth_d.callback(True)
def error_cb(re):
found_cb()
def_search = Deferred()
find_instagram_config = {"@id": "service_instagram_config"}
logging.info(
"Searching for instagram_config to check if Popular feed already harvested... "
)
def_search = self.indx_con.query(json.dumps(find_instagram_config))
def_search.addCallbacks(found_cb, error_cb)
return auth_d
# for feed in user_feed:
# print feed
def get_searched_media(self, search_terms):
print "getting searched media for terms: " + str(search_terms)
returned_results, page_num = self.api.tag_search(search_terms, 20)
return returned_results
# for result in returned_results:
# print result
def get_popular_media(self):
pop_d = Deferred()
def found_cb(results):
friends_number = 0
followers_number = 0
since_id = ""
#let's see if the object has some nice things in it.
try:
config_returned = results['data']['service_instagram_config']
friends_number = int(
config_returned['friends_list_size'][0]['@value'])
followers_number = int(
config_returned['followers_list_size'][0]['@value'])
since_id = int(config_returned['since_id'][0]['@value'])
logging.info('Found the instagram Config Object.')
except:
#print sys.exc_info()
pass
#if(since_id > 0):
logging.info("getting popular media")
objects_to_insert = []
current_timestamp = str(
time.time()).split(".")[0] #str(datetime.now())
timestamp = str(datetime.now().isoformat('T')).split(".")[0]
current_timestamp = str(datetime.now())
popular_media = self.api.media_popular(count=20)
for media in popular_media:
media_user_id = media.user.id
media_username = media.user.username
#now create the instagram_user object
uniq_user_id = "instagram_user_" + media.user.id
instagram_media_obj = {
"@id": uniq_user_id,
"app_object": appid,
"timestamp": timestamp,
"type": "user",
"instagram_user_id": media_user_id,
"instagram_username": media_username
}
objects_to_insert.append(instagram_media_obj)
uniq_id = "instagram_media_" + media.id
media_obj = {
"@id": uniq_id,
"app_object": appid,
"timestamp": timestamp,
"type": "post",
"instagram_user_id_indx": uniq_user_id,
"instagram_user_id": media_user_id,
"instagram_username": media_username,
"media_id": media.id,
"media_caption": media.caption,
"media_url": media.get_standard_resolution_url(),
"media_like_count": media.like_count
}
#need to add INDX objects for tags and locations and the indx user of this
#create location if available
try:
location = media.location
uniq_id = "instagram_location_" + location.id
location_obj = {
"@id": uniq_id,
"app_object": appid,
"timestamp": timestamp,
"type": "location",
"instagram_location_id": location.id,
"location_name": location.name,
"latitude": location.point.latitude,
"longitude": location.point.longitude
}
#now add this location to the user_feed_obj
media_obj['media_location_id'] = location_obj
#thhen add it to a list of ojects to insert.
objects_to_insert.append(location_obj)
except:
pass
try:
tag_ids = []
for tag in media.tags:
uniq_id = "instagram_tag_" + tag.name
tag_obj = {
"@id": uniq_id,
"app_object": appid,
"type": "tag",
"timestamp": timestamp,
"instagram_tag_name": tag.name
}
tag_ids.append(uniq_id)
objects_to_insert.append(tag_obj)
#now add this location to the user_feed_obj
media_obj['tags'] = tag_ids
except:
pass
objects_to_insert.append(media_obj)
def update_cb(re):
logging.debug("network harvest async worked {0}".format(re))
pop_d.callback(True)
def update_cb_fail(re):
logging.error("network harvest async failed {0}".format(re))
pop_d.errback
#and create the instagram config
instagram_config_obj = {
"@id": "service_instagram_config",
"app_object": appid,
"type": "config",
"config_last_updated_at": timestamp,
"config_for_instagram_user": self.instagram_username,
"friends_list_generated_at": timestamp,
"follower_list_generated_at": timestamp,
"friends_list_size": friends_number,
"followers_list_size": followers_number,
"since_id": since_id
}
objects_to_insert.append(instagram_config_obj)
self.insert_object_to_indx(objects_to_insert).addCallbacks(
update_cb, update_cb_fail)
def error_cb(re):
found_cb()
def_search = Deferred()
find_instagram_config = {"@id": "service_instagram_config"}
logging.info(
"Searching for instagram_config to check if Popular feed already harvested... "
)
def_search = self.indx_con.query(json.dumps(find_instagram_config))
def_search.addCallbacks(found_cb, error_cb)
return pop_d
def find_user(self, username):
data = self.api.user_search(username, count=20)
print data
def find_followers(self, userid):
#first do a lookup and see if the latest set if followers has allready been found
follower_d = Deferred()
def found_cb(results):
friends_number = 0
followers_number = 0
since_id = 0
#let's see if the object has some nice things in it.
try:
config_returned = results['data']['service_instagram_config']
friends_number = int(
config_returned['friends_list_size'][0]['@value'])
followers_number = int(
config_returned['followers_list_size'][0]['@value'])
since_id = int(config_returned['since_id'][0]['@value'])
logging.info('Found the instagram Config Object.')
except:
#print sys.exc_info()
pass
followed_by = self.api.user_followed_by(userid)[0]
#print "followed_by length: "+str(len(followed_by))
#print str(followed_by)
#see if the number is different, if it is, then update.. could be nicer than this, but for now, it will work (ish)
if (len(followed_by) != followers_number) or (followers_number
== 0):
followers_number = len(followed_by)
objects_to_insert = []
current_timestamp = str(
time.time()).split(".")[0] #str(datetime.now())
timestamp = str(datetime.now().isoformat('T')).split(".")[0]
uniq_id = "instagram_follower_network_for_user_me" #+self.instagram_username
followed_by_obj = {
"@id": uniq_id,
"app_object": appid,
"instagram_username": self.instagram_username,
"instagram_user_id": self.instagram_user_id,
"timestamp": timestamp,
"followed_by_count": len(followed_by)
}
followers_ids = []
for follower in followed_by:
#print follower.username
uniq_id = "instagram_user_" + str(follower.username)
follower_obj = {
"@id": uniq_id,
"app_object": appid,
"type": "user",
"instagram_username": str(follower.username),
"timestamp": timestamp
}
objects_to_insert.append(follower_obj)
#we can add this to the followed_by_obj later
followers_ids.append(uniq_id)
#link the followers for me
followed_by_obj['follower_ids'] = followers_ids
# print followed_by_objs
#for friend in friends_list:
#print friend
#now append the results
def update_cb(re):
logging.debug(
"network harvest async worked {0}".format(re))
follower_d.callback(True)
def update_cb_fail(re):
logging.error(
"network harvest async failed {0}".format(re))
follower_d.errback
instagram_config_obj = {
"@id": "service_instagram_config",
"app_object": appid,
"type": "config",
"config_last_updated_at": timestamp,
"config_for_instagram_user": self.instagram_username,
"friends_list_generated_at": timestamp,
"follower_list_generated_at": timestamp,
"friends_list_size": friends_number,
"followers_list_size": followers_number,
"since_id": since_id
}
objects_to_insert.append(followed_by_obj)
#objects_to_insert.append(followers)
objects_to_insert.append(instagram_config_obj)
self.insert_object_to_indx(objects_to_insert).addCallbacks(
update_cb, update_cb_fail)
else:
follower_d.callback(True)
def error_cb(re):
found_cb()
def_search = Deferred()
find_instagram_config = {"@id": "service_instagram_config"}
logging.info(
"Searching for instagram_config to check if network already harvested... "
)
def_search = self.indx_con.query(json.dumps(find_instagram_config))
def_search.addCallbacks(found_cb, error_cb)
return follower_d
# print followed_by
def find_friends(self, userid):
friends_d = Deferred()
def found_cb(results):
friends_number = 0
followers_number = 0
since_id = 0
#let's see if the object has some nice things in it.
try:
config_returned = results['data']['service_instagram_config']
friends_number = int(
config_returned['friends_list_size'][0]['@value'])
followers_number = int(
config_returned['followers_list_size'][0]['@value'])
since_id = int(config_returned['since_id'][0]['@value'])
logging.info('Found the instagram Config Object.')
except:
#print sys.exc_info()
pass
friends_by_list = self.api.user_follows(userid)[0]
if (len(friends_by_list) != friends_number) or (friends_number
== 0):
friends_number = len(friends_by_list)
objects_to_insert = []
current_timestamp = str(
time.time()).split(".")[0] #str(datetime.now())
timestamp = str(datetime.now().isoformat('T')).split(".")[0]
uniq_id = "instagram_friends_network_for_user_me" #+self.instagram_username
friends_by_obj = {
"@id": uniq_id,
"app_object": appid,
"instagram_username": self.instagram_username,
"instagram_user_id": self.instagram_user_id,
"timestamp": timestamp,
"followed_by_count": len(friends_by_list)
}
friends_ids = []
for friend in friends_by_list:
#print follower.username
uniq_id = "instagram_user_" + str(friend.username)
friend_obj = {
"@id": uniq_id,
"app_object": appid,
"type": "user",
"instagram_username": str(friend.username),
"timestamp": timestamp
}
objects_to_insert.append(friend_obj)
#we can add this to the followed_by_obj later
friends_ids.append(uniq_id)
friends_by_obj['friends_ids'] = friends_ids
# print friends_by_objs
#for friend in friends_list:
#print friend
#now append the results
def update_cb(re):
logging.debug(
"network harvest async worked {0}".format(re))
friends_d.callback(True)
def update_cb_fail(re):
logging.error(
"network harvest async failed {0}".format(re))
friends_d.errback
instagram_config_obj = {
"@id": "service_instagram_config",
"app_object": appid,
"type": "config",
"config_last_updated_at": timestamp,
"config_for_instagram_user": self.instagram_username,
"friends_list_generated_at": timestamp,
"follower_list_generated_at": timestamp,
"friends_list_size": friends_number,
"followers_list_size": followers_number,
"since_id": since_id
}
objects_to_insert.append(friends_by_obj)
#objects_to_insert.append(followers)
objects_to_insert.append(instagram_config_obj)
self.insert_object_to_indx(objects_to_insert).addCallbacks(
update_cb, update_cb_fail)
else:
friends_d.callback(True)
def error_cb(re):
found_cb()
def_search = Deferred()
find_instagram_config = {"@id": "service_instagram_config"}
logging.info(
"Searching for instagram_config to check if network already harvested... "
)
def_search = self.indx_con.query(json.dumps(find_instagram_config))
def_search.addCallbacks(found_cb, error_cb)
return friends_d
# print followed_by
def subscribe_to_objects_by_tag(self, search_terms):
def process_tag_update(update):
print update
reactor = subscriptions.SubscriptionsReactor()
reactor.register_callback(subscriptions.SubscriptionType.TAG,
process_tag_update)
self.api.create_subscription(object='tag',
object_id=search_terms,
aspect='media',
callback_url='http://*****:*****@version']
logging.debug(
"Succesfully Updated INDX with Objects in update_cb, new diff version of {0}"
.format(self.version))
#logging.debug("Inserted Object into INDX: ".format(resp))
update_d.callback(True)
#return update_d
#time.sleep(2)
def exception_cb(e, service=self, obj=obj):
logging.debug(
"Exception Inserting into INDX, probably wrong version given")
if isinstance(
e.value, urllib2.HTTPError
): # handle a version incorrect error, and update the version
if e.value.code == 409: # 409 Obsolete
response = e.value.read()
json_response = json.loads(response)
service.version = json_response['@version']
logging.debug(
'INDX insert error in Instagram Service Object: ' +
str(response))
try:
service.indx_con.update(service.version,
obj).addCallbacks(
update_cb, exception_cb)
#logging.debug('Twitter Service - Successfully added Objects into Box')
except:
logging.error(
'Instagram Service, error on insert {0}'.format(
response))
update_d.errback(e.value)
else:
logging.error('Instagram Service Unknow error: {0}'.format(
e.value.read()))
update_d.errback(e.value)
else:
logging.error("Error updating INDX: {0}".format(e.value))
update_d.errback(e.value)
logging.debug(
"in Instagram - Trying to insert into indx...Current diff version: {0} and objects (len) given {1}"
.format(self.version, len(obj)))
self.indx_con.update(self.version,
obj).addCallbacks(update_cb, exception_cb)
return update_d
def token_cb(token):
self.indx_con = IndxClient(self.config['address'], self.config['box'], app_id, token = token, client = authclient.client)
return_d.callback(True)
class FacebookService:
def __init__(self, config):
logging.debug('Facebook Service - Getting configs from Facebook Controller')
self.config = config
self.facebook_access_token_long = config['facebook_access_token_long']
self.facebook_userid = config['facebook_userid']
self.facebook_access_token_expire_time = config['facebook_access_token_expire_time']
self.config_timestamp = config['config_timestamp']
self.version = 0
#in reality this is all we need from facebook, but we want to also check that the current date is not over 60 days...
logging.debug('Facebook Service - Checking if token is not expired')
self.token_active = False
if self.check_if_token_not_expired():
logging.info('Facebook Service - Token is active still. Great News!')
self.token_active = True
def get_indx(self):
return_d = Deferred()
def authed_cb():
logging.debug("in service_tweets - Authed Callback")
logging.debug("in service_tweets - get_indx authclient Auth status: {0}".format(authclient.is_authed))
def token_cb(token):
self.indx_con = IndxClient(self.config['address'], self.config['box'], app_id, token = token, client = authclient.client)
return_d.callback(True)
authclient.get_token(self.config['box']).addCallbacks(token_cb, return_d.errback)
def authed_cb_fail(re):
logging.debug("in Facebook Service - get_indx/authed_cb failed for reason {0}".format(re))
return_d.errback
logging.debug("in Facebook Service - get_indx")
authclient = IndxClientAuth(self.config['address'], app_id)
authclient.auth_plain(self.config['user'], self.config['password']).addCallbacks(lambda response: authed_cb(), authed_cb_fail)
return return_d
# def get_indx(self):
# return_d = Deferred()
# # self.indx_con = IndxClient(self.config['address'], self.config['box'], self.config['user'], self.config['password'], app_id)
# def authed_cb():
# def token_cb(token):
# self.indx_con = IndxClient(self.config['address'], self.config['box'], app_id, token = token)
# return_d.callback(True)
# authclient.get_token(self.config['box']).addCallbacks(token_cb, return_d.errback)
# authclient = IndxClientAuth(self.config['address'], app_id)
# authclient.auth_plain(self.config['user'], self.config['password']).addCallbacks(lambda response: authed_cb(), return_d.errback)
# return return_d
def is_token_active(self):
return self.token_active
#need to check if the token has already expired, we have around 60 days. Returns true if token still active
def check_if_token_not_expired(self):
current_time = datetime.datetime.now()
access_token_time = (self.config_timestamp).split(".")[0] # we just want it as yyyy-mm-ddT:hh:mm:ss
access_token_time = self.createTimestamp(access_token_time);
delta = current_time - access_token_time
delta_days = delta.days
if (delta_days) < 60:
return True
else:
return False
#this should be a library function somewhere...
def createTimestamp(self,strTimestamp):
try:
return datetime.datetime.strptime(strTimestamp, '%Y-%m-%dT%H:%S:%M')
except:
if not ":" in str(strTimestamp):
strTimestamp = strTimestamp+" 00:00:00"
return datetime.datetime.strptime(strTimestamp, '%Y-%m-%d %H:%S:%M')
else:
pass
def harvest_all(self):
harvest_d = Deferred()
if(self.is_token_active()):
logging.debug("Service Controller Facebook - Running Facebook Service!")
def havest_friends_cb(re):
logging.debug("Service Controller Facebook - Updated Profile")
def havest_statuses_cb(res):
logging.debug("Service Controller Facebook - Updated Status {0}".format(res))
harvest_d.callback(True)
# def havest_profile_cb(resu):
# logging.debug("Service Controller Facebook - Updated Friends")
# #update every hour
# #reactor.callLater(3600.0, loop_harvester);
# harvest_d.callback(True)
# self.harvest_facebook_profile().addCallbacks(havest_profile_cb, lambda failure: logging.error("Facebook Service Controller - Callback Failure: Harvest Profile"))
self.harvest_facebook_statuses().addCallbacks(havest_statuses_cb, lambda failure: logging.error("Facebook Service Controller - Callback Failure: Harvest Statuses"))
self.harvest_facebook_friends().addCallbacks(havest_friends_cb, lambda failure: logging.error("Facebook Service Controller - Callback Failure: Harvest Friends"))
return harvest_d
##here are all the facebook methods to havest the data
def harvest_facebook_profile(self):
harvest_profile_d = Deferred()
logging.info('Facebook Service - Getting users Facebook Profile')
graph = GraphAPI(self.facebook_access_token_long)
profile = graph.get_object("me")
timestamp = str(datetime.datetime.now().isoformat('T')).split(".")[0]
uniq_id = "facebook_profile_for_me"
object_to_insert = {"@id":uniq_id, "app_object": app_id, "timestamp": timestamp, "facebook_profile": profile}
#now need to perform the asnc
def insert_cb(re):
logging.debug("Facebook Service - Found Facebook Profile information, Added To INDX {0} profile items".format(len(profile)))
harvest_profile_d.callback(True)
def insert_cb_fail(re):
harvest_profile_d.callback(True)
self.insert_object_to_indx(object_to_insert).addCallbacks(insert_cb, insert_cb_fail)
return harvest_profile_d
##here are all the facebook methods to havest the data
def harvest_facebook_statuses(self):
harvest_status_d = Deferred()
def found_cb(results):
friends_number = 0
followers_number = 0
since_id = 0
#let's see if the object has some nice things in it.
try:
config_returned = results['data']['service_facebook_config']
friends_number = int(config_returned['friends_list_size'][0]['@value'])
followers_number = int(config_returned['followers_list_size'][0]['@value'])
since_id = int(config_returned['since_id'][0]['@value'])
logging.debug('Found the Facebook Config Object.')
except:
#print sys.exc_info()
pass
logging.info('Facebook Service - Getting users Facebook Statuses')
graph = GraphAPI(self.facebook_access_token_long)
profile = graph.get_object("me")
if len(profile)>1:
#print "got Profile so should be able to get status"
facebook_id = str(profile['id'])
facebook_username = str(profile['name'])
query = facebook_id#+"/permissions"
statuses = graph.get_connections(facebook_id, "statuses", limit=10000) #graph.get_object(query) #
timestamp = str(datetime.datetime.now().isoformat('T')).split(".")[0]
statuses = statuses['data']
#need to make sure that we are not fetching statuses already added...
latest_status_id = 0
try:
latest_status_id = statuses[0]['id']
except:
pass
if (latest_status_id > since_id) or (latest_status_id==0):
since_id = latest_status_id
objects_to_insert = []
for status in statuses:
uniq_id = "facebook_statuses_"+str(status['id'])
facebook_status_obj = {"@id":uniq_id, "app_object": app_id, "timestamp":timestamp, "type": "post", "facebook_status_id":status['id'],
"facebook_user_indx": "facebook_user_me", "facebook_user_id":facebook_id}
#for each status, get the time
facebook_status_obj['facebook_status_text'] = status['message']
facebook_status_obj['facebook_status_timestamp'] = str(status['updated_time']).split("+")[0]
like_ids = []
try:
likes = status['likes']['data']
totalLikes = len(likes)
facebook_status_obj['facebook_status_total_likes'] = str(totalLikes)
for like in likes:
#construct the facebook user that liked your post
uniq_id = "facebook_user_"+str(like['id'])
facebook_user_obj = {"@id":uniq_id, "app_object": app_id, "timestamp":timestamp, "type": "user", "facebook_username": like['name'], "facebook_id":like['id']}
like_ids.append(uniq_id)
objects_to_insert.append(facebook_user_obj)
except:
pass
comment_ids = []
try:
comments = status['comments']['data']
totalcomments = len(comments)
facebook_status_obj['facebook_status_total_comments'] = str(totalcomments)
for comment in comments:
#first create the comment user
facebook_comment_obj_id = "facebook_user_"+str(comment['from']['id'])
facebook_user_obj = {"@id":facebook_comment_obj_id, "app_object": app_id, "timestamp":timestamp, "type": "user", "facebook_username": comment['from']['name'], "facebook_id":comment['from']['id']}
#add it to the indx objects
objects_to_insert.append(facebook_user_obj)
#create the facebook comment object
uniq_id = "facebook_comment_"+comment['id']
facebook_comment_obj = {"@id":uniq_id, "app_object": app_id, "timestamp":timestamp, "type": "post", "facebook_user_indx": facebook_comment_obj_id,
"facebook_id":comment['from']['id'], "facebook_username":comment['from']['name']}
#for each status, get the time
facebook_comment_obj['facebook_comment_text'] = comment['message']
facebook_comment_obj['facebook_comment_timestamp'] = str(comment['created_time']).split("+")[0]
facebook_comment_obj['like_count'] = comment['like_count']
#add it to the indx objects
objects_to_insert.append(facebook_comment_obj)
#for reference
comment_ids.append(uniq_id)
except:
pass
facebook_status_obj['facebook_like_user_ids_indx'] = like_ids
facebook_status_obj['facebook_comment_user_ids_indx'] = comment_ids
#add it to the indx objects
objects_to_insert.append(facebook_status_obj)
#then create the user object
uniq_id = "facebook_user_me"
facebook_user_obj = {"@id":uniq_id, "app_object": app_id, "timestamp":timestamp, "type": "user", "facebook_username": facebook_username, "facebook_id":facebook_id}
objects_to_insert.append(facebook_user_obj)
#then add all of the extra info
#and create the instagram config
facebook_config_obj = {"@id": "service_facebook_config", "app_object": app_id, "type":"config", "config_last_updated_at": timestamp,
"config_for_facebook_user_id": self.facebook_userid, "friends_list_generated_at": timestamp, "follower_list_generated_at": timestamp,
"friends_list_size":friends_number, "followers_list_size": followers_number, "since_id":since_id}
objects_to_insert.append(facebook_config_obj)
#now need to perform the asnc
def insert_cb(re):
logging.info("Facebook Service - Found {0} statuses, Added To INDX".format(len(statuses)))
harvest_status_d.callback(True)
def insert_cb_fail(re):
harvest_status_d.callback(True)
logging.debug("inserting statuses into indx")
self.insert_object_to_indx(objects_to_insert).addCallbacks(insert_cb, insert_cb_fail)
else:
harvest_status_d.callback(True)
def error_cb(re):
found_cb()
def_search = Deferred()
find_instagram_config = {"@id": "service_facebook_config"}
logging.info("Searching for facebook_config to check if latest statues already harvested... ")
def_search = self.indx_con.query(json.dumps(find_instagram_config))
def_search.addCallbacks(found_cb, error_cb)
return harvest_status_d
def harvest_facebook_friends(self):
harvest_friends_d = Deferred()
def found_cb(results):
friends_number = 0
followers_number = 0
since_id = 0
#let's see if the object has some nice things in it.
try:
config_returned = results['data']['service_facebook_config']
friends_number = int(config_returned['friends_list_size'][0]['@value'])
followers_number = int(config_returned['followers_list_size'][0]['@value'])
since_id = int(config_returned['since_id'][0]['@value'])
logging.debug('Found the Facebook Config Object.')
except:
#print sys.exc_info()
pass
logging.info('Facebook Service - Getting users Facebook Friends')
graph = GraphAPI(self.facebook_access_token_long)
#profile = graph.get_object("me")
#rint profile
friends_all = graph.get_connections("me", "friends", limit=1000)
friends = friends_all['data']
friends_number = len(friends)
logging.debug('Facebook Service - Got users Facebook Friends: {0}'.format(friends_number))
objects_to_insert = []
timestamp = str(datetime.datetime.now().isoformat('T')).split(".")[0]
friend_ids = []
counter = 0
friends_objs_to_insert = []
for friend in friends:
uniq_id = "facebook_user_"+str(friend['id'])
friend_obj = {"@id":uniq_id, "app_object": app_id, "timestamp":timestamp, "type": "user", "facebook_user_id": friend['id'], "facebook_username": friend['name']}
friend_ids.append(uniq_id)
friends_objs_to_insert.append(friend_obj)
try:
#now create the me facebook user friend object
profile = graph.get_object("me")
facebook_id = str(profile['id'])
facebook_username = str(profile['name'])
uniq_id = "facebook_friends_for_user_me"
facebook_user_friend_obj = {"@id":uniq_id, "app_object": app_id, "timestamp":timestamp, "faceboob_user_id_indx": "facebook_user_me",
"facebook_user_id": facebook_id, "facebook_username": facebook_username, "facebook_friends_ids_indx": friend_ids}
objects_to_insert.append(facebook_user_friend_obj)
except:
pass
#now set the config
facebook_config_obj = {"@id": "service_facebook_config", "app_object": app_id, "type":"config", "config_last_updated_at": timestamp,
"config_for_facebook_user_id": facebook_id, "friends_list_generated_at": timestamp, "follower_list_generated_at": timestamp,
"friends_list_size":friends_number, "followers_list_size": followers_number, "since_id":since_id}
objects_to_insert.append(facebook_config_obj)
def insertfriends(self):
if (len(friends_objs_to_insert) > 0):
def insert_friend_cb(re):
insertfriends(self)
def insert_friend_cb_fail(re):
insertfriends(self)
friends_to_add = []
pop_point = 0
for x in range(0, 100):
try:
friends_to_add.append(friends_objs_to_insert[x])
pop_point += 1
except:
pass
for x in range(0, pop_point):
try:
friends_objs_to_insert.pop(0)
except:
pass
#friend_object = friends_objs_to_insert[0]
#print "friends list length: "+str(len(friends_objs_to_insert))
self.insert_object_to_indx(friends_to_add).addCallbacks(insert_friend_cb, insert_friend_cb_fail)
else:
logging.info("All Facebook Friends Added to Indx, will now proceed.")
#this is a trick as indxclient is a bit nasty with big inserts
logging.debug("Inserting friends into indx")
insertfriends(self)
#now need to perform the asnc
def insert_cb(re):
logging.debug("Facebook Service - Found Friends List, Added To INDX {0} Friends".format(len(friends)))
harvest_friends_d.callback(True)
def insert_cb_fail(re):
harvest_friends_d.callback(True)
logging.debug("inserting friends config into indx")
self.insert_object_to_indx(objects_to_insert).addCallbacks(insert_cb, insert_cb_fail)
def error_cb(re):
found_cb()
def_search = Deferred()
find_facebook_config = {"@id": "service_facebook_config"}
logging.info("Searching for facebook_config to check if Popular feed already harvested... ")
def_search = self.indx_con.query(json.dumps(find_facebook_config))
def_search.addCallbacks(found_cb, error_cb)
return harvest_friends_d
def insert_object_to_indx(self, obj):
update_d = Deferred()
def update_cb(resp):
self.version = resp['data']['@version']
logging.debug("Succesfully Updated INDX with Objects in update_cb, new diff version of {0}".format(self.version))
#logging.debug("Inserted Object into INDX: ".format(resp))
update_d.callback(True)
#return update_d
#time.sleep(2)
def exception_cb(e, service=self, obj=obj):
logging.debug("Exception Inserting into INDX, probably wrong version given")
if isinstance(e.value, urllib2.HTTPError): # handle a version incorrect error, and update the version
if e.value.code == 409: # 409 Obsolete
response = e.value.read()
json_response = json.loads(response)
service.version = json_response['@version']
logging.debug('INDX insert error in Instagram Service Object: '+str(response))
try:
service.indx_con.update(service.version, obj).addCallbacks(update_cb, exception_cb)
#logging.debug('Twitter Service - Successfully added Objects into Box')
except:
logging.debug('Instagram Service, error on insert {0}'.format(response))
update_d.errback(e.value)
else:
logging.debug('Instagram Service Unknow error: {0}'.format(e.value.read()))
update_d.errback(e.value)
else:
logging.error("Error updating INDX: {0}".format(e.value))
update_d.errback(e.value)
logging.debug("in Instagram - Trying to insert into indx...Current diff version: {0} and objects (len) given {1}".format(self.version, len(obj)))
self.indx_con.update(self.version, obj).addCallbacks(update_cb, exception_cb)
return update_d
class FacebookService:
def __init__(self, config):
logging.debug(
'Facebook Service - Getting configs from Facebook Controller')
self.config = config
self.facebook_access_token_long = config['facebook_access_token_long']
self.facebook_userid = config['facebook_userid']
self.facebook_access_token_expire_time = config[
'facebook_access_token_expire_time']
self.config_timestamp = config['config_timestamp']
self.version = 0
#in reality this is all we need from facebook, but we want to also check that the current date is not over 60 days...
logging.debug('Facebook Service - Checking if token is not expired')
self.token_active = False
if self.check_if_token_not_expired():
logging.info(
'Facebook Service - Token is active still. Great News!')
self.token_active = True
def get_indx(self):
return_d = Deferred()
def authed_cb():
logging.debug("in service_tweets - Authed Callback")
logging.debug(
"in service_tweets - get_indx authclient Auth status: {0}".
format(authclient.is_authed))
def token_cb(token):
self.indx_con = IndxClient(self.config['address'],
self.config['box'],
app_id,
token=token,
client=authclient.client)
return_d.callback(True)
authclient.get_token(self.config['box']).addCallbacks(
token_cb, return_d.errback)
def authed_cb_fail(re):
logging.debug(
"in Facebook Service - get_indx/authed_cb failed for reason {0}"
.format(re))
return_d.errback
logging.debug("in Facebook Service - get_indx")
authclient = IndxClientAuth(self.config['address'], app_id)
authclient.auth_plain(self.config['user'],
self.config['password']).addCallbacks(
lambda response: authed_cb(), authed_cb_fail)
return return_d
# def get_indx(self):
# return_d = Deferred()
# # self.indx_con = IndxClient(self.config['address'], self.config['box'], self.config['user'], self.config['password'], app_id)
# def authed_cb():
# def token_cb(token):
# self.indx_con = IndxClient(self.config['address'], self.config['box'], app_id, token = token)
# return_d.callback(True)
# authclient.get_token(self.config['box']).addCallbacks(token_cb, return_d.errback)
# authclient = IndxClientAuth(self.config['address'], app_id)
# authclient.auth_plain(self.config['user'], self.config['password']).addCallbacks(lambda response: authed_cb(), return_d.errback)
# return return_d
def is_token_active(self):
return self.token_active
#need to check if the token has already expired, we have around 60 days. Returns true if token still active
def check_if_token_not_expired(self):
current_time = datetime.datetime.now()
access_token_time = (self.config_timestamp).split(".")[
0] # we just want it as yyyy-mm-ddT:hh:mm:ss
access_token_time = self.createTimestamp(access_token_time)
delta = current_time - access_token_time
delta_days = delta.days
if (delta_days) < 60:
return True
else:
return False
#this should be a library function somewhere...
def createTimestamp(self, strTimestamp):
try:
return datetime.datetime.strptime(strTimestamp,
'%Y-%m-%dT%H:%S:%M')
except:
if not ":" in str(strTimestamp):
strTimestamp = strTimestamp + " 00:00:00"
return datetime.datetime.strptime(strTimestamp,
'%Y-%m-%d %H:%S:%M')
else:
pass
def harvest_all(self):
harvest_d = Deferred()
if (self.is_token_active()):
logging.debug(
"Service Controller Facebook - Running Facebook Service!")
def havest_friends_cb(re):
logging.debug("Service Controller Facebook - Updated Profile")
def havest_statuses_cb(res):
logging.debug(
"Service Controller Facebook - Updated Status {0}".
format(res))
harvest_d.callback(True)
# def havest_profile_cb(resu):
# logging.debug("Service Controller Facebook - Updated Friends")
# #update every hour
# #reactor.callLater(3600.0, loop_harvester);
# harvest_d.callback(True)
# self.harvest_facebook_profile().addCallbacks(havest_profile_cb, lambda failure: logging.error("Facebook Service Controller - Callback Failure: Harvest Profile"))
self.harvest_facebook_statuses().addCallbacks(
havest_statuses_cb, lambda failure: logging.error(
"Facebook Service Controller - Callback Failure: Harvest Statuses"
))
self.harvest_facebook_friends().addCallbacks(
havest_friends_cb, lambda failure: logging.error(
"Facebook Service Controller - Callback Failure: Harvest Friends"
))
return harvest_d
##here are all the facebook methods to havest the data
def harvest_facebook_profile(self):
harvest_profile_d = Deferred()
logging.info('Facebook Service - Getting users Facebook Profile')
graph = GraphAPI(self.facebook_access_token_long)
profile = graph.get_object("me")
timestamp = str(datetime.datetime.now().isoformat('T')).split(".")[0]
uniq_id = "facebook_profile_for_me"
object_to_insert = {
"@id": uniq_id,
"app_object": app_id,
"timestamp": timestamp,
"facebook_profile": profile
}
#now need to perform the asnc
def insert_cb(re):
logging.debug(
"Facebook Service - Found Facebook Profile information, Added To INDX {0} profile items"
.format(len(profile)))
harvest_profile_d.callback(True)
def insert_cb_fail(re):
harvest_profile_d.callback(True)
self.insert_object_to_indx(object_to_insert).addCallbacks(
insert_cb, insert_cb_fail)
return harvest_profile_d
##here are all the facebook methods to havest the data
def harvest_facebook_statuses(self):
harvest_status_d = Deferred()
def found_cb(results):
friends_number = 0
followers_number = 0
since_id = 0
#let's see if the object has some nice things in it.
try:
config_returned = results['data']['service_facebook_config']
friends_number = int(
config_returned['friends_list_size'][0]['@value'])
followers_number = int(
config_returned['followers_list_size'][0]['@value'])
since_id = int(config_returned['since_id'][0]['@value'])
logging.debug('Found the Facebook Config Object.')
except:
#print sys.exc_info()
pass
logging.info('Facebook Service - Getting users Facebook Statuses')
graph = GraphAPI(self.facebook_access_token_long)
profile = graph.get_object("me")
if len(profile) > 1:
#print "got Profile so should be able to get status"
facebook_id = str(profile['id'])
facebook_username = str(profile['name'])
query = facebook_id #+"/permissions"
statuses = graph.get_connections(
facebook_id, "statuses",
limit=10000) #graph.get_object(query) #
timestamp = str(
datetime.datetime.now().isoformat('T')).split(".")[0]
statuses = statuses['data']
#need to make sure that we are not fetching statuses already added...
latest_status_id = 0
try:
latest_status_id = statuses[0]['id']
except:
pass
if (latest_status_id > since_id) or (latest_status_id == 0):
since_id = latest_status_id
objects_to_insert = []
for status in statuses:
uniq_id = "facebook_statuses_" + str(status['id'])
facebook_status_obj = {
"@id": uniq_id,
"app_object": app_id,
"timestamp": timestamp,
"type": "post",
"facebook_status_id": status['id'],
"facebook_user_indx": "facebook_user_me",
"facebook_user_id": facebook_id
}
#for each status, get the time
facebook_status_obj['facebook_status_text'] = status[
'message']
facebook_status_obj['facebook_status_timestamp'] = str(
status['updated_time']).split("+")[0]
like_ids = []
try:
likes = status['likes']['data']
totalLikes = len(likes)
facebook_status_obj[
'facebook_status_total_likes'] = str(
totalLikes)
for like in likes:
#construct the facebook user that liked your post
uniq_id = "facebook_user_" + str(like['id'])
facebook_user_obj = {
"@id": uniq_id,
"app_object": app_id,
"timestamp": timestamp,
"type": "user",
"facebook_username": like['name'],
"facebook_id": like['id']
}
like_ids.append(uniq_id)
objects_to_insert.append(facebook_user_obj)
except:
pass
comment_ids = []
try:
comments = status['comments']['data']
totalcomments = len(comments)
facebook_status_obj[
'facebook_status_total_comments'] = str(
totalcomments)
for comment in comments:
#first create the comment user
facebook_comment_obj_id = "facebook_user_" + str(
comment['from']['id'])
facebook_user_obj = {
"@id": facebook_comment_obj_id,
"app_object": app_id,
"timestamp": timestamp,
"type": "user",
"facebook_username":
comment['from']['name'],
"facebook_id": comment['from']['id']
}
#add it to the indx objects
objects_to_insert.append(facebook_user_obj)
#create the facebook comment object
uniq_id = "facebook_comment_" + comment['id']
facebook_comment_obj = {
"@id": uniq_id,
"app_object": app_id,
"timestamp": timestamp,
"type": "post",
"facebook_user_indx":
facebook_comment_obj_id,
"facebook_id": comment['from']['id'],
"facebook_username":
comment['from']['name']
}
#for each status, get the time
facebook_comment_obj[
'facebook_comment_text'] = comment[
'message']
facebook_comment_obj[
'facebook_comment_timestamp'] = str(
comment['created_time']).split("+")[0]
facebook_comment_obj['like_count'] = comment[
'like_count']
#add it to the indx objects
objects_to_insert.append(facebook_comment_obj)
#for reference
comment_ids.append(uniq_id)
except:
pass
facebook_status_obj[
'facebook_like_user_ids_indx'] = like_ids
facebook_status_obj[
'facebook_comment_user_ids_indx'] = comment_ids
#add it to the indx objects
objects_to_insert.append(facebook_status_obj)
#then create the user object
uniq_id = "facebook_user_me"
facebook_user_obj = {
"@id": uniq_id,
"app_object": app_id,
"timestamp": timestamp,
"type": "user",
"facebook_username": facebook_username,
"facebook_id": facebook_id
}
objects_to_insert.append(facebook_user_obj)
#then add all of the extra info
#and create the instagram config
facebook_config_obj = {
"@id": "service_facebook_config",
"app_object": app_id,
"type": "config",
"config_last_updated_at": timestamp,
"config_for_facebook_user_id": self.facebook_userid,
"friends_list_generated_at": timestamp,
"follower_list_generated_at": timestamp,
"friends_list_size": friends_number,
"followers_list_size": followers_number,
"since_id": since_id
}
objects_to_insert.append(facebook_config_obj)
#now need to perform the asnc
def insert_cb(re):
logging.info(
"Facebook Service - Found {0} statuses, Added To INDX".
format(len(statuses)))
harvest_status_d.callback(True)
def insert_cb_fail(re):
harvest_status_d.callback(True)
logging.debug("inserting statuses into indx")
self.insert_object_to_indx(objects_to_insert).addCallbacks(
insert_cb, insert_cb_fail)
else:
harvest_status_d.callback(True)
def error_cb(re):
found_cb()
def_search = Deferred()
find_instagram_config = {"@id": "service_facebook_config"}
logging.info(
"Searching for facebook_config to check if latest statues already harvested... "
)
def_search = self.indx_con.query(json.dumps(find_instagram_config))
def_search.addCallbacks(found_cb, error_cb)
return harvest_status_d
def harvest_facebook_friends(self):
harvest_friends_d = Deferred()
def found_cb(results):
friends_number = 0
followers_number = 0
since_id = 0
#let's see if the object has some nice things in it.
try:
config_returned = results['data']['service_facebook_config']
friends_number = int(
config_returned['friends_list_size'][0]['@value'])
followers_number = int(
config_returned['followers_list_size'][0]['@value'])
since_id = int(config_returned['since_id'][0]['@value'])
logging.debug('Found the Facebook Config Object.')
except:
#print sys.exc_info()
pass
logging.info('Facebook Service - Getting users Facebook Friends')
graph = GraphAPI(self.facebook_access_token_long)
#profile = graph.get_object("me")
#rint profile
friends_all = graph.get_connections("me", "friends", limit=1000)
friends = friends_all['data']
friends_number = len(friends)
logging.debug(
'Facebook Service - Got users Facebook Friends: {0}'.format(
friends_number))
objects_to_insert = []
timestamp = str(
datetime.datetime.now().isoformat('T')).split(".")[0]
friend_ids = []
counter = 0
friends_objs_to_insert = []
for friend in friends:
uniq_id = "facebook_user_" + str(friend['id'])
friend_obj = {
"@id": uniq_id,
"app_object": app_id,
"timestamp": timestamp,
"type": "user",
"facebook_user_id": friend['id'],
"facebook_username": friend['name']
}
friend_ids.append(uniq_id)
friends_objs_to_insert.append(friend_obj)
try:
#now create the me facebook user friend object
profile = graph.get_object("me")
facebook_id = str(profile['id'])
facebook_username = str(profile['name'])
uniq_id = "facebook_friends_for_user_me"
facebook_user_friend_obj = {
"@id": uniq_id,
"app_object": app_id,
"timestamp": timestamp,
"faceboob_user_id_indx": "facebook_user_me",
"facebook_user_id": facebook_id,
"facebook_username": facebook_username,
"facebook_friends_ids_indx": friend_ids
}
objects_to_insert.append(facebook_user_friend_obj)
except:
pass
#now set the config
facebook_config_obj = {
"@id": "service_facebook_config",
"app_object": app_id,
"type": "config",
"config_last_updated_at": timestamp,
"config_for_facebook_user_id": facebook_id,
"friends_list_generated_at": timestamp,
"follower_list_generated_at": timestamp,
"friends_list_size": friends_number,
"followers_list_size": followers_number,
"since_id": since_id
}
objects_to_insert.append(facebook_config_obj)
def insertfriends(self):
if (len(friends_objs_to_insert) > 0):
def insert_friend_cb(re):
insertfriends(self)
def insert_friend_cb_fail(re):
insertfriends(self)
friends_to_add = []
pop_point = 0
for x in range(0, 100):
try:
friends_to_add.append(friends_objs_to_insert[x])
pop_point += 1
except:
pass
for x in range(0, pop_point):
try:
friends_objs_to_insert.pop(0)
except:
pass
#friend_object = friends_objs_to_insert[0]
#print "friends list length: "+str(len(friends_objs_to_insert))
self.insert_object_to_indx(friends_to_add).addCallbacks(
insert_friend_cb, insert_friend_cb_fail)
else:
logging.info(
"All Facebook Friends Added to Indx, will now proceed."
)
#this is a trick as indxclient is a bit nasty with big inserts
logging.debug("Inserting friends into indx")
insertfriends(self)
#now need to perform the asnc
def insert_cb(re):
logging.debug(
"Facebook Service - Found Friends List, Added To INDX {0} Friends"
.format(len(friends)))
harvest_friends_d.callback(True)
def insert_cb_fail(re):
harvest_friends_d.callback(True)
logging.debug("inserting friends config into indx")
self.insert_object_to_indx(objects_to_insert).addCallbacks(
insert_cb, insert_cb_fail)
def error_cb(re):
found_cb()
def_search = Deferred()
find_facebook_config = {"@id": "service_facebook_config"}
logging.info(
"Searching for facebook_config to check if Popular feed already harvested... "
)
def_search = self.indx_con.query(json.dumps(find_facebook_config))
def_search.addCallbacks(found_cb, error_cb)
return harvest_friends_d
def insert_object_to_indx(self, obj):
update_d = Deferred()
def update_cb(resp):
self.version = resp['data']['@version']
logging.debug(
"Succesfully Updated INDX with Objects in update_cb, new diff version of {0}"
.format(self.version))
#logging.debug("Inserted Object into INDX: ".format(resp))
update_d.callback(True)
#return update_d
#time.sleep(2)
def exception_cb(e, service=self, obj=obj):
logging.debug(
"Exception Inserting into INDX, probably wrong version given")
if isinstance(
e.value, urllib2.HTTPError
): # handle a version incorrect error, and update the version
if e.value.code == 409: # 409 Obsolete
response = e.value.read()
json_response = json.loads(response)
service.version = json_response['@version']
logging.debug(
'INDX insert error in Instagram Service Object: ' +
str(response))
try:
service.indx_con.update(service.version,
obj).addCallbacks(
update_cb, exception_cb)
#logging.debug('Twitter Service - Successfully added Objects into Box')
except:
logging.debug(
'Instagram Service, error on insert {0}'.format(
response))
update_d.errback(e.value)
else:
logging.debug('Instagram Service Unknow error: {0}'.format(
e.value.read()))
update_d.errback(e.value)
else:
logging.error("Error updating INDX: {0}".format(e.value))
update_d.errback(e.value)
logging.debug(
"in Instagram - Trying to insert into indx...Current diff version: {0} and objects (len) given {1}"
.format(self.version, len(obj)))
self.indx_con.update(self.version,
obj).addCallbacks(update_cb, exception_cb)
return update_d
class CLIClient:
def __init__(self, appid = "INDX CLIClient"):
""" Associate command-line actions with functions, and enforce required variables. """
self.actions = {'create_box': {'f': self.create_box, 'args': ['box']},
'delete_box': {'f': self.delete_box, 'args': ['box']},
'list_boxes': {'f': self.list_boxes, 'args': []},
'get_object_ids': {'f': self.get_object_ids, 'args': ['box']},
'create_user': {'f': self.create_user, 'args': ['target_username','target_password']},
'update': {'f': self.update, 'args': ['box','data','version']},
'delete': {'f': self.delete, 'args': ['box','id','version']},
'get_latest': {'f': self.get_latest, 'args': ['box']},
'get_by_ids': {'f': self.get_by_ids, 'args': ['box','id']},
'query': {'f': self.query, 'args': ['box','query']},
'diff': {'f': self.diff, 'args': ['box','from','return_objs']},
#'listen': {'f': self.listen, 'args': ['box']},
'add_file': {'f': self.add_file, 'args': ['box','data','id','version','contenttype']},
'delete_file': {'f': self.delete_file, 'args': ['box','id','version']},
'get_file': {'f': self.get_file, 'args': ['box','id']},
'list_files': {'f': self.list_files, 'args': ['box']},
'set_acl': {'f': self.set_acl, 'args': ['box','acl','target_username']},
'get_acls': {'f': self.get_acls, 'args': ['box']},
'generate_new_key': {'f': self.generate_new_key, 'args': ['box']},
'create_root_box': {'f': self.create_root_box, 'args': ['box']},
'link_remote_box': {'f': self.link_remote_box, 'args': ['box', 'remote_token', 'remote_box', 'remote_address']},
'generate_token': {'f': self.generate_token, 'args': ['box']},
}
self.token = None
self.appid = appid
self.indx = None
def set_args(self, args):
""" Move relevant command-line arguments into local variable. """
logging.debug("Set args: {0}".format(args))
self.args = args
""" Flatten single value lists into flat key/value pair. """
for key in args:
if type(args[key]) == type([]) and len(args[key]) == 1:
args[key] = args[key][0]
""" Ensure self.server always ends in a / """
if self.args['server'][-1:] != "/":
self.args['server'] += "/"
def check_args(self, required):
not_set = []
for key in required:
if key not in self.args or self.args[key] is None or self.args[key] == "":
not_set.append(key)
if not self.args['allowempty'] and len(not_set) > 0:
raise Exception("The following values cannot be empty for this action: {0}".format(", ".join(not_set)))
def auth_and_get_token(self, get_token):
""" Authenticate, get a token and call it back to the deferred. """
return_d = Deferred()
def authed_cb():
def token_cb(token):
if token is not None:
self.token = token
return_d.callback(token)
else:
return_d.callback(None)
if get_token:
authclient.get_token(self.args['box']).addCallbacks(token_cb, return_d.errback)
else:
token_cb(None)
authclient = IndxClientAuth(self.args['server'], self.appid)
self.client = authclient.client
authclient.auth_plain(self.args['username'], self.args['password']).addCallbacks(lambda response: authed_cb(), return_d.errback)
return return_d
def call_action(self, name, *args, **kwargs):
""" Calls an action by name. """
return_d = Deferred()
action = self.actions[name]
f = action['f']
self.check_args(action['args'])
def do_call():
f(*args, **kwargs).addCallbacks(lambda status: return_d.callback(self.parse_status(name, status)), return_d.errback)
if not self.token:
def token_cb(token):
if not self.indx:
self.indx = IndxClient(self.args['server'], self.args['box'], self.appid, token = token, client = self.client)
do_call()
self.auth_and_get_token(IndxClient.requires_token(f)).addCallbacks(token_cb, return_d.errback)
else:
do_call()
return return_d
def parse_status(self, source, status):
""" Parse the status returned from the server, and raise an Exception if necessary. """
if status is not None: # status is None when a file has been printed raw
if status['code'] < 200 or status['code'] > 299:
raise Exception("{0} in box {1} failed. Response is {2} with code {3}".format(source, self.args['box'], status['message'], status['code']))
else:
if "data" in status and self.args['jsondata']:
return json.dumps(status['data'], indent = 2)
else:
pretty = pprint.pformat(status, indent=2, width=80)
logging.info("{0} in box {1} successful, return is: {2}".format(source, self.args['box'], pretty))
return None
""" Test functions."""
def create_box(self):
""" Test to create a box. """
logging.debug("Creating box: '{0}' on server '{1}'".format(self.args['box'], self.args['server']))
return self.indx.create_box()
def delete_box(self):
""" Test to delete a box. """
logging.debug("Deleting box: '{0}' on server '{1}'".format(self.args['box'], self.args['server']))
return self.indx.delete_box()
def list_boxes(self):
""" List the boxes on the INDX server. """
logging.debug("Listing boxes on server '{0}'".format(self.args['server']))
return self.indx.list_boxes()
def get_object_ids(self):
""" Get the IDs of every object in this box. """
logging.debug("Getting a list of object IDs on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
return self.indx.get_object_ids()
def update(self):
""" Test to update objects in a box. """
logging.debug("Updating data to box: '{0}' on server '{1}'".format(self.args['box'], self.args['server']))
#return self.indx.update(self.args['version'], cjson.decode(self.args['data'].read()), all_unicode=True)
return self.indx.update(self.args['version'], json.loads(self.args['data'].read()))
def delete(self):
""" Test to delete objects from a box.
'id' argument should be a list of ids
e.g., --id=id1 --id=id2
"""
logging.debug("Deleting data to box: '{0}' on server '{1}'".format(self.args['box'], self.args['server']))
return self.indx.delete(self.args['version'], self.args['id'])
def get_latest(self):
""" Get the latest version of every object in this box. """
logging.debug("Getting latest objects on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
return self.indx.get_latest()
def get_by_ids(self):
""" Get the latest version of specific objects in this box. """
logging.debug("Getting latest objects on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
ids = self.args['id']
if type(ids) != type([]):
ids = [ids] # put a single id into an array
return self.indx.get_by_ids(ids)
def query(self):
""" Query this box.
e.g., query="{ '@id': 2983 }"
or query="{ 'firstname': 'dan' }"
"""
logging.debug("Querying server '{0}' in box '{1}' with depth '{2}'".format(self.args['server'], self.args['box'], self.args['depth']))
return self.indx.query(self.args['query'], depth = int(self.args['depth']))
def diff(self):
""" Run a diff of two version of this box. """
logging.debug("Calling diff on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
to_version = None
if "to" in self.args and self.args['to'] is not None:
to_version = self.args['to']
return self.indx.diff(self.args['return_objs'], self.args['from'], to_version = to_version)
# def listen(self):
# """ Listen to updates to the database (locally, not with HTTP) and print out the diff in realtime. """
# self.check_args(['box', 'username', 'password'])
#
# def observer(notify):
# print "Version updated to: {0}".format(notify.payload)
#
# def err_cb(failure):
# logging.error("Error in test listen: {0}".format(failure))
# reactor.stop()
#
# def connected_cb(conn):
# print "Listening..."
# conns = {"conn": conn}
# store = ObjectStoreAsync(conns, self.args['username'], self.appid, "127.0.0.1") # TODO get the IP a better way? does it matter here?
# store.listen(observer)
#
# d = database.connect_box_raw(self.args['box'], self.args['username'], self.args['password'])
# d.addCallbacks(connected_cb, err_cb)
# reactor.run()
def add_file(self):
""" Add a file to the database. """
logging.debug("Adding a file to server '{0}' in box '{1}', with id: {2}".format(self.args['server'], self.args['box'], self.args['id']))
return self.indx.add_file(self.args['version'], self.args['id'], self.args['data'].read(), self.args['contenttype'])
def delete_file(self):
""" Delete a file from the database. """
logging.debug("Deleting a file from server '{0}' in box '{1}', with id: {2}".format(self.args['server'], self.args['box'], self.args['id']))
return self.indx.delete_file(self.args['version'], self.args['id'])
def get_file(self):
""" Get a file from the database. """
logging.debug("Calling get_file on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
# returns the file to stdout so it can be piped to a file
print self.indx.get_file(self.args['id'])
def list_files(self):
""" Get a list of the files from the database. """
logging.debug("Calling list_files on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
return self.indx.list_files()
def set_acl(self):
""" Set an ACL for a target user for a database. """
logging.debug("Calling set_acl on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
return self.indx.set_acl(self.args['acl'], self.args['target_username'])
def get_acls(self):
""" Get ACLs for a database. """
logging.debug("Calling get_acls on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
return self.indx.get_acls()
def generate_new_key(self):
""" Generate and store a new key, returning the public and public-hash parts of the key. """
logging.debug("Calling generate_new_key on server '{0}' in box '{1}'".format(self.args['server'], self.args['box']))
return self.indx.generate_new_key()
def create_root_box(self):
""" Create root box for a user. """
logging.debug("Calling create_root_box on server '{0}' for box '{1}'".format(self.args['server'], self.args['box']))
return self.indx.create_root_box(self.args['box'])
def create_user(self):
""" Create a new user. """
logging.debug("Calling create_user on server '{0}' with target username '{1}'".format(self.args['server'], self.args['target_username']))
return self.indx.create_user(self.args['target_username'], self.args['target_password'])
def link_remote_box(self):
""" Link a remote box with a local box. """
logging.debug("Calling link_remote_box on remote_address '{0}', remote_box '{1}', remote_token '{2}'".format(self.args['remote_address'], self.args['remote_box'], self.args['remote_token']))
return self.indx.link_remote_box(self.args['remote_address'], self.args['remote_box'], self.args['remote_token'])
def generate_token(self):
""" Generate a token for this box, and print it. """
logging.debug("Calling generate_token on box {0}.".format(self.args['box']))
return_d = Deferred()
def token_cb(token):
return_d.callback({"code": 200, "data": token})
self.auth_and_get_token(True).addCallbacks(token_cb, return_d.errback)
return return_d
class instagramService:
def __init__(self, config):
self.config = config
self.api = InstagramAPI(access_token=config["access_token"])
self.version = 0
self.batch = []
self.batch_users = []
self.feed_count = 0
self.feed_count_total = 0
self.instagram_username = config["instagram_username"]
self.instagram_user_id = config["instagram_user_id"]
def stop_and_exit_service(self):
logging.debug("instagram Service - HARD QUIT - instagram SERVICE")
sys.exit()
def run_main_services(self):
main_services_d = Deferred()
try:
print "running instagram main services"
def find_followers_cb(res):
def find_friends_cb(res):
def get_authen_user_feed_cb(res):
def get_popular_media_cb(res):
main_services_d.callback(True)
self.get_popular_media().addCallbacks(
get_popular_media_cb, lambda failure: logging.error("Update Error {0}".format(failure))
)
self.get_authenticated_user_feed().addCallbacks(
get_authen_user_feed_cb, lambda failure: logging.error("Update Error {0}".format(failure))
)
self.find_friends(self.config["instagram_user_id"]).addCallbacks(
find_friends_cb, lambda failure: logging.error("Update Error{0}".format(failure))
)
self.find_followers(self.config["instagram_user_id"]).addCallbacks(
find_followers_cb, lambda failure: logging.error("Update Error{0}".format(failure))
)
# self.subscribe_to_objects_by_tag(self.config['instagram_search_words'])
except:
logging.debug("Service Instagram - Could not run main service due to error: {0}".format(sys.exc_info()))
return main_services_d
def get_indx(self):
return_d = Deferred()
def authed_cb():
logging.debug("in service_tweets - Authed Callback")
logging.debug("in service_tweets - get_indx authclient Auth status: {0}".format(authclient.is_authed))
def token_cb(token):
self.indx_con = IndxClient(
self.config["address"], self.config["box"], appid, token=token, client=authclient.client
)
return_d.callback(True)
authclient.get_token(self.config["box"]).addCallbacks(token_cb, return_d.errback)
def authed_cb_fail(re):
logging.debug("in service tweets - get_indx/authed_cb failed for reason {0}".format(re))
return_d.errback
logging.debug("in service_instagram - get_indx")
authclient = IndxClientAuth(self.config["address"], appid)
authclient.auth_plain(self.config["user"], self.config["password"]).addCallbacks(
lambda response: authed_cb(), authed_cb_fail
)
return return_d
def get_authenticated_user_feed(self):
auth_d = Deferred()
def found_cb(results):
friends_number = 0
followers_number = 0
since_id = ""
# let's see if the object has some nice things in it.
try:
config_returned = results["data"]["service_instagram_config"]
friends_number = int(config_returned["friends_list_size"][0]["@value"])
followers_number = int(config_returned["followers_list_size"][0]["@value"])
since_id = int(config_returned["since_id"][0]["@value"])
logging.info("Found the instagram Config Object.")
except:
# print sys.exc_info()
pass
# print "Getting Auth User's feed"
user_feed = self.api.user_media_feed()[0]
try:
latest_id = user_feed[0].id
except:
latest_id = "Null"
##find the highest id...
if latest_id != since_id:
logging.info("Found some new media, will insert it to INDX")
since_id = latest_id
objects_to_insert = []
current_timestamp = str(time.time()).split(".")[0] # str(datetime.now())
timestamp = str(datetime.now().isoformat("T")).split(".")[0]
current_timestamp = str(datetime.now())
# the user responsible for this
# now create some nice user objects...
for media in user_feed:
media_user_id = media.user.id
media_username = media.user.username
uniq_id = "instagram_media_" + media.id
user_feed_obj = {
"@id": uniq_id,
"app_object": appid,
"timestamp": timestamp,
"type": "post",
"instagram_user_id_indx": "instagram_user_me",
"instagram_user_id": media_user_id,
"instagram_username": media_username,
"media_id": media.id,
"media_caption": media.caption,
"media_url": media.get_standard_resolution_url(),
"media_like_count": media.like_count,
}
# need to add INDX objects for tags and locations and the indx user of this
# create location if available
try:
location = media.location
uniq_id = "instagram_location_" + location.id
location_obj = {
"@id": uniq_id,
"app_object": appid,
"timestamp": timestamp,
"type": "location",
"instagram_location_id": location.id,
"location_name": location.name,
"latitude": location.point.latitude,
"longitude": location.point.longitude,
}
# now add this location to the user_feed_obj
user_feed_obj["media_location_id"] = location_obj
# thhen add it to a list of ojects to insert.
objects_to_insert.append(location_obj)
except:
pass
try:
tag_ids = []
for tag in media.tags:
uniq_id = "instagram_tag_" + tag.name
tag_obj = {
"@id": uniq_id,
"app_object": appid,
"timestamp": timestamp,
"type": "tag",
"instagram_tag_name": tag.name,
}
tag_ids.append(uniq_id)
objects_to_insert.append(tag_obj)
# now add this location to the user_feed_obj
user_feed_obj["tags"] = tag_ids
except:
pass
objects_to_insert.append(user_feed_obj)
# now create the instagram_user_me object
uniq_id = "instagram_user_me"
instagram_me_obj = {
"@id": uniq_id,
"app_object": appid,
"timestamp": timestamp,
"type": "user",
"instagram_user_id": media_user_id,
"instagram_username": media_username,
}
objects_to_insert.append(instagram_me_obj)
# and create the instagram config
instagram_config_obj = {
"@id": "service_instagram_config",
"app_object": appid,
"type": "config",
"config_last_updated_at": timestamp,
"config_for_instagram_user": self.instagram_username,
"friends_list_generated_at": timestamp,
"follower_list_generated_at": timestamp,
"friends_list_size": friends_number,
"followers_list_size": followers_number,
"since_id": since_id,
}
objects_to_insert.append(instagram_config_obj)
def update_cb(re):
logging.debug("network harvest async worked {0}".format(re))
auth_d.callback(True)
def update_cb_fail(re):
logging.error("network harvest async failed {0}".format(re))
auth_d.errback
self.insert_object_to_indx(objects_to_insert).addCallbacks(update_cb, update_cb_fail)
else:
logging.info("already have the latest version of your instagram timeline")
auth_d.callback(True)
def error_cb(re):
found_cb()
def_search = Deferred()
find_instagram_config = {"@id": "service_instagram_config"}
logging.info("Searching for instagram_config to check if Popular feed already harvested... ")
def_search = self.indx_con.query(json.dumps(find_instagram_config))
def_search.addCallbacks(found_cb, error_cb)
return auth_d
# for feed in user_feed:
# print feed
def get_searched_media(self, search_terms):
print "getting searched media for terms: " + str(search_terms)
returned_results, page_num = self.api.tag_search(search_terms, 20)
return returned_results
# for result in returned_results:
# print result
def get_popular_media(self):
pop_d = Deferred()
def found_cb(results):
friends_number = 0
followers_number = 0
since_id = ""
# let's see if the object has some nice things in it.
try:
config_returned = results["data"]["service_instagram_config"]
friends_number = int(config_returned["friends_list_size"][0]["@value"])
followers_number = int(config_returned["followers_list_size"][0]["@value"])
since_id = int(config_returned["since_id"][0]["@value"])
logging.info("Found the instagram Config Object.")
except:
# print sys.exc_info()
pass
# if(since_id > 0):
logging.info("getting popular media")
objects_to_insert = []
current_timestamp = str(time.time()).split(".")[0] # str(datetime.now())
timestamp = str(datetime.now().isoformat("T")).split(".")[0]
current_timestamp = str(datetime.now())
popular_media = self.api.media_popular(count=20)
for media in popular_media:
media_user_id = media.user.id
media_username = media.user.username
# now create the instagram_user object
uniq_user_id = "instagram_user_" + media.user.id
instagram_media_obj = {
"@id": uniq_user_id,
"app_object": appid,
"timestamp": timestamp,
"type": "user",
"instagram_user_id": media_user_id,
"instagram_username": media_username,
}
objects_to_insert.append(instagram_media_obj)
uniq_id = "instagram_media_" + media.id
media_obj = {
"@id": uniq_id,
"app_object": appid,
"timestamp": timestamp,
"type": "post",
"instagram_user_id_indx": uniq_user_id,
"instagram_user_id": media_user_id,
"instagram_username": media_username,
"media_id": media.id,
"media_caption": media.caption,
"media_url": media.get_standard_resolution_url(),
"media_like_count": media.like_count,
}
# need to add INDX objects for tags and locations and the indx user of this
# create location if available
try:
location = media.location
uniq_id = "instagram_location_" + location.id
location_obj = {
"@id": uniq_id,
"app_object": appid,
"timestamp": timestamp,
"type": "location",
"instagram_location_id": location.id,
"location_name": location.name,
"latitude": location.point.latitude,
"longitude": location.point.longitude,
}
# now add this location to the user_feed_obj
media_obj["media_location_id"] = location_obj
# thhen add it to a list of ojects to insert.
objects_to_insert.append(location_obj)
except:
pass
try:
tag_ids = []
for tag in media.tags:
uniq_id = "instagram_tag_" + tag.name
tag_obj = {
"@id": uniq_id,
"app_object": appid,
"type": "tag",
"timestamp": timestamp,
"instagram_tag_name": tag.name,
}
tag_ids.append(uniq_id)
objects_to_insert.append(tag_obj)
# now add this location to the user_feed_obj
media_obj["tags"] = tag_ids
except:
pass
objects_to_insert.append(media_obj)
def update_cb(re):
logging.debug("network harvest async worked {0}".format(re))
pop_d.callback(True)
def update_cb_fail(re):
logging.error("network harvest async failed {0}".format(re))
pop_d.errback
# and create the instagram config
instagram_config_obj = {
"@id": "service_instagram_config",
"app_object": appid,
"type": "config",
"config_last_updated_at": timestamp,
"config_for_instagram_user": self.instagram_username,
"friends_list_generated_at": timestamp,
"follower_list_generated_at": timestamp,
"friends_list_size": friends_number,
"followers_list_size": followers_number,
"since_id": since_id,
}
objects_to_insert.append(instagram_config_obj)
self.insert_object_to_indx(objects_to_insert).addCallbacks(update_cb, update_cb_fail)
def error_cb(re):
found_cb()
def_search = Deferred()
find_instagram_config = {"@id": "service_instagram_config"}
logging.info("Searching for instagram_config to check if Popular feed already harvested... ")
def_search = self.indx_con.query(json.dumps(find_instagram_config))
def_search.addCallbacks(found_cb, error_cb)
return pop_d
def find_user(self, username):
data = self.api.user_search(username, count=20)
print data
def find_followers(self, userid):
# first do a lookup and see if the latest set if followers has allready been found
follower_d = Deferred()
def found_cb(results):
friends_number = 0
followers_number = 0
since_id = 0
# let's see if the object has some nice things in it.
try:
config_returned = results["data"]["service_instagram_config"]
friends_number = int(config_returned["friends_list_size"][0]["@value"])
followers_number = int(config_returned["followers_list_size"][0]["@value"])
since_id = int(config_returned["since_id"][0]["@value"])
logging.info("Found the instagram Config Object.")
except:
# print sys.exc_info()
pass
followed_by = self.api.user_followed_by(userid)[0]
# print "followed_by length: "+str(len(followed_by))
# print str(followed_by)
# see if the number is different, if it is, then update.. could be nicer than this, but for now, it will work (ish)
if (len(followed_by) != followers_number) or (followers_number == 0):
followers_number = len(followed_by)
objects_to_insert = []
current_timestamp = str(time.time()).split(".")[0] # str(datetime.now())
timestamp = str(datetime.now().isoformat("T")).split(".")[0]
uniq_id = "instagram_follower_network_for_user_me" # +self.instagram_username
followed_by_obj = {
"@id": uniq_id,
"app_object": appid,
"instagram_username": self.instagram_username,
"instagram_user_id": self.instagram_user_id,
"timestamp": timestamp,
"followed_by_count": len(followed_by),
}
followers_ids = []
for follower in followed_by:
# print follower.username
uniq_id = "instagram_user_" + str(follower.username)
follower_obj = {
"@id": uniq_id,
"app_object": appid,
"type": "user",
"instagram_username": str(follower.username),
"timestamp": timestamp,
}
objects_to_insert.append(follower_obj)
# we can add this to the followed_by_obj later
followers_ids.append(uniq_id)
# link the followers for me
followed_by_obj["follower_ids"] = followers_ids
# print followed_by_objs
# for friend in friends_list:
# print friend
# now append the results
def update_cb(re):
logging.debug("network harvest async worked {0}".format(re))
follower_d.callback(True)
def update_cb_fail(re):
logging.error("network harvest async failed {0}".format(re))
follower_d.errback
instagram_config_obj = {
"@id": "service_instagram_config",
"app_object": appid,
"type": "config",
"config_last_updated_at": timestamp,
"config_for_instagram_user": self.instagram_username,
"friends_list_generated_at": timestamp,
"follower_list_generated_at": timestamp,
"friends_list_size": friends_number,
"followers_list_size": followers_number,
"since_id": since_id,
}
objects_to_insert.append(followed_by_obj)
# objects_to_insert.append(followers)
objects_to_insert.append(instagram_config_obj)
self.insert_object_to_indx(objects_to_insert).addCallbacks(update_cb, update_cb_fail)
else:
follower_d.callback(True)
def error_cb(re):
found_cb()
def_search = Deferred()
find_instagram_config = {"@id": "service_instagram_config"}
logging.info("Searching for instagram_config to check if network already harvested... ")
def_search = self.indx_con.query(json.dumps(find_instagram_config))
def_search.addCallbacks(found_cb, error_cb)
return follower_d
# print followed_by
def find_friends(self, userid):
friends_d = Deferred()
def found_cb(results):
friends_number = 0
followers_number = 0
since_id = 0
# let's see if the object has some nice things in it.
try:
config_returned = results["data"]["service_instagram_config"]
friends_number = int(config_returned["friends_list_size"][0]["@value"])
followers_number = int(config_returned["followers_list_size"][0]["@value"])
since_id = int(config_returned["since_id"][0]["@value"])
logging.info("Found the instagram Config Object.")
except:
# print sys.exc_info()
pass
friends_by_list = self.api.user_follows(userid)[0]
if (len(friends_by_list) != friends_number) or (friends_number == 0):
friends_number = len(friends_by_list)
objects_to_insert = []
current_timestamp = str(time.time()).split(".")[0] # str(datetime.now())
timestamp = str(datetime.now().isoformat("T")).split(".")[0]
uniq_id = "instagram_friends_network_for_user_me" # +self.instagram_username
friends_by_obj = {
"@id": uniq_id,
"app_object": appid,
"instagram_username": self.instagram_username,
"instagram_user_id": self.instagram_user_id,
"timestamp": timestamp,
"followed_by_count": len(friends_by_list),
}
friends_ids = []
for friend in friends_by_list:
# print follower.username
uniq_id = "instagram_user_" + str(friend.username)
friend_obj = {
"@id": uniq_id,
"app_object": appid,
"type": "user",
"instagram_username": str(friend.username),
"timestamp": timestamp,
}
objects_to_insert.append(friend_obj)
# we can add this to the followed_by_obj later
friends_ids.append(uniq_id)
friends_by_obj["friends_ids"] = friends_ids
# print friends_by_objs
# for friend in friends_list:
# print friend
# now append the results
def update_cb(re):
logging.debug("network harvest async worked {0}".format(re))
friends_d.callback(True)
def update_cb_fail(re):
logging.error("network harvest async failed {0}".format(re))
friends_d.errback
instagram_config_obj = {
"@id": "service_instagram_config",
"app_object": appid,
"type": "config",
"config_last_updated_at": timestamp,
"config_for_instagram_user": self.instagram_username,
"friends_list_generated_at": timestamp,
"follower_list_generated_at": timestamp,
"friends_list_size": friends_number,
"followers_list_size": followers_number,
"since_id": since_id,
}
objects_to_insert.append(friends_by_obj)
# objects_to_insert.append(followers)
objects_to_insert.append(instagram_config_obj)
self.insert_object_to_indx(objects_to_insert).addCallbacks(update_cb, update_cb_fail)
else:
friends_d.callback(True)
def error_cb(re):
found_cb()
def_search = Deferred()
find_instagram_config = {"@id": "service_instagram_config"}
logging.info("Searching for instagram_config to check if network already harvested... ")
def_search = self.indx_con.query(json.dumps(find_instagram_config))
def_search.addCallbacks(found_cb, error_cb)
return friends_d
# print followed_by
def subscribe_to_objects_by_tag(self, search_terms):
def process_tag_update(update):
print update
reactor = subscriptions.SubscriptionsReactor()
reactor.register_callback(subscriptions.SubscriptionType.TAG, process_tag_update)
self.api.create_subscription(
object="tag", object_id=search_terms, aspect="media", callback_url="http://*****:*****@version"]
logging.debug(
"Succesfully Updated INDX with Objects in update_cb, new diff version of {0}".format(self.version)
)
# logging.debug("Inserted Object into INDX: ".format(resp))
update_d.callback(True)
# return update_d
# time.sleep(2)
def exception_cb(e, service=self, obj=obj):
logging.debug("Exception Inserting into INDX, probably wrong version given")
if isinstance(e.value, urllib2.HTTPError): # handle a version incorrect error, and update the version
if e.value.code == 409: # 409 Obsolete
response = e.value.read()
json_response = json.loads(response)
service.version = json_response["@version"]
logging.debug("INDX insert error in Instagram Service Object: " + str(response))
try:
service.indx_con.update(service.version, obj).addCallbacks(update_cb, exception_cb)
# logging.debug('Twitter Service - Successfully added Objects into Box')
except:
logging.error("Instagram Service, error on insert {0}".format(response))
update_d.errback(e.value)
else:
logging.error("Instagram Service Unknow error: {0}".format(e.value.read()))
update_d.errback(e.value)
else:
logging.error("Error updating INDX: {0}".format(e.value))
update_d.errback(e.value)
logging.debug(
"in Instagram - Trying to insert into indx...Current diff version: {0} and objects (len) given {1}".format(
self.version, len(obj)
)
)
self.indx_con.update(self.version, obj).addCallbacks(update_cb, exception_cb)
return update_d
