def authenticate(self, auth_token=None): if auth_token is None: from oauth2client.client import OAuth2WebServerFlow self.flow = OAuth2WebServerFlow( getSecrets('client_id'), getSecrets('client_secret'), getSecrets('scopes'), getSecrets('redirect_uri')) return self.flow.step1_get_authorize_url() else: credentials = self.flow.step2_exchange(auth_token) auth_storage = os.path.join(CONF_ROOT, "drive.secrets.json") from oauth2client.file import Storage Storage(auth_storage).put(credentials) del self.flow from conf import setSecrets return setSecrets({ 'auth_storage' : auth_storage, 'account_type' : "user" }) return False
def register_upload_attempt(_id): res = None cmd = "cd %s && .git/hooks/uv-on-upload-attempted \"%s\"" % ( getSecrets('annex_remote'), _id) print "REGISTERING UPLOAD ATTEMPT" USE_SSH = getSecrets('server_force_ssh') if USE_SSH is None: if SERVER_HOST in ["127.0.0.1", "localhost"]: USE_SSH = False else: USE_SSH = True if USE_SSH: env.key_filename = [getSecrets('ssh_key_pub').replace(".pub", "")] cmd = "ssh -f -p %d -i %s %s -o IdentitiesOnly=yes 'source ~/.bash_profile && %s'" % ( getSecrets('annex_remote_port'), env.key_filename[0], env.host_string.split(":")[0], cmd) with settings(warn_only=True): res = local(cmd, capture=True) return res
def __init__(self, func, args=None, op_dir=None): self.func = func if args is not None: self.args = args else: self.args = {} self.output = None self.error = None USE_SSH = getSecrets("server_force_ssh") if USE_SSH is None: if SERVER_HOST in ["127.0.0.1", "localhost"]: USE_SSH = False else: USE_SSH = True if USE_SSH: uv_user = getSecrets("server_user") hostname = getSecrets("server_host") port = getSecrets("annex_remote_port") port_prefix = "" if port is not None and port != 22: port_prefix += ":%d" % port self.args.update({"hosts": ["%s@%s%s" % (uv_user, hostname, port_prefix)]}) if op_dir is not None: self.return_dir = os.getcwd() os.chdir(op_dir) threading.Thread.__init__(self) self.start()
def checkAnnexKeyStatus(self, send_key=True): if getSecrets('server_host') == "127.0.0.1": return True if send_key and getSecrets('annex_admin_email') is not None: pub_key = self.upload(getSecrets('ssh_key_pub'), title="annex public key") if pub_key is not None and self.share(pub_key['id']) is not None: return True return False
def notarize(self): if hasattr(self, "notarized") and self.notarized: print "ALREADY NOTARIZED: (%s)" % self.notarized return False if not self.get_provenance(): print "NO PROVENANCE YET" return False j3m = self.getAsset("j3m_raw.json", return_only="path") if j3m is None: print "NO RAW J3M TO NOTARIZE." return False from lib.Core.Utils.funcs import hashEntireFile j3m_hash = hashEntireFile(j3m, hfunc="sha256") if j3m_hash is None: print "WHY DIDN'T HASH WORK?" return False import gnupg from vars import ASSET_TAGS from conf import getConfig, getSecrets try: gpg = gnupg.GPG(homedir=getConfig('gpg_homedir')) gpg_pwd = getSecrets('gpg_pwd') notarization_doc = gpg.sign(j3m_hash, \ default_key=getSecrets('org_fingerprint')[-8:], passphrase=gpg_pwd) del gpg_pwd if len(notarization_doc.data) == 0: print "NO NOTARIZATION DOC" return False self.addAsset(notarization_doc.data, "j3m_notarized.asc", tags=[ASSET_TAGS['IC_NOTARIZE']], description="Signed Notarization Doc, ready to publish.") self.notarized = True self.saveFields("notarized") return True except Exception as e: print "PROBLEM SIGNING DOC:" print e, type(e) return False
def do_reindex(self, handler): req = parseRequestEntity(handler.request.body) if req is None: return None if '_id' not in req.keys(): return None ''' we might have to replace parts of the request with secret values. ''' if 'task_path' in req.keys(): req_keys = list( chain.from_iterable( [k.keys() for k in MIME_TYPE_TASK_REQUIREMENTS])) if req['task_path'] in req_keys: try: req_field = [ k for k in MIME_TYPE_TASK_REQUIREMENTS if k.keys()[0] == req['task_path'] ][0][req['task_path']] print req_field except Exception as e: if DEBUG: print "Could not resolve required task info. %s" % e for repl in req_field.iteritems(): if repl[0] in req.keys(): handler.request.body = handler.request.body.replace( repl[1], getSecrets(repl[1])) return self.passToAnnex(handler)
def post_to_slack(self, file_name, file_content, bot_callback=None, title=None): slack = getSecrets('slack') data = { 'token' : slack['api_token'], 'title' : "i deepdreamed..." if title is None else title, 'channels' : slack['channel_id'] } try: r = requests.post("https://slack.com/api/files.upload", data=data, files={ 'file' : file_content}) res = json.loads(r.content) if not res['ok']: return False if bot_callback is not None: r = requests.post(slack['webhook_url'], data={'payload' : json.dumps({ 'text' : bot_callback })}) print r.content return True except Exception as e: print e, type(e) return False
def exportFrontendConfig(with_config=None, with_secrets=None): import json from conf import DEBUG, SERVER_HOST, UUID, ANNEX_DIR, API_PORT, getConfig, SHA1_INDEX server_message_port = None try: server_message_port = getConfig('server_message_port') except: pass server_user = None try: server_user = getConfig('server_user') except: with settings(warn_only=True): server_user = local('whoami', capture=True) config = { 'server_host' : SERVER_HOST, 'server_port' : API_PORT, 'annex_remote' : ANNEX_DIR, 'uv_uuid' : UUID, 'annex_remote_port' : 22, 'server_use_ssl' : False, 'gdrive_auth_no_ask' : True, 'server_message_port' : (API_PORT + 1) if server_message_port is None else server_message_port, 'server_user' : server_user, 'index.sha1' : SHA1_INDEX } if with_config is not None: if type(with_config) is not list: with_config = [with_config] for c in with_config: try: config[c] = getConfig(c) except exception as e: if DEBUG: print e if with_secrets is not None: from conf import getSecrets if type(with_secrets) is not list: with_secrets = [with_secrets] for s in with_secrets: try: config[s] = getSecrets(s) except exception as e: if DEBUG: print e for key in [k for k in config.keys() if config[k] is None]: del config[key] print "***********************************************" print "THE FOLLOWING LINES MAKE FOR YOUR FRONTEND CONFIG\n\n" print json.dumps(config) print "***********************************************" return config
def do_reindex(self, handler): req = parseRequestEntity(handler.request.body) if req is None: return None if '_id' not in req.keys(): return None ''' we might have to replace parts of the request with secret values. ''' if 'task_path' in req.keys(): req_keys = list(chain.from_iterable( [k.keys() for k in TASK_REQUIREMENTS])) if req['task_path'] in req_keys: try: req_field = [k for k in TASK_REQUIREMENTS if k.keys()[0] == req['task_path']][0][req['task_path']] print req_field except Exception as e: if DEBUG: print "Could not resolve required task info. %s" % e for repl in req_field.iteritems(): if repl[0] in req.keys(): handler.request.body = handler.request.body.replace( repl[1], getSecrets(repl[1])) return self.passToAnnex(handler)
def post_to_slack(self, file_name, file_content, bot_callback=None, title=None): slack = getSecrets('slack') data = { 'token': slack['api_token'], 'title': "i deepdreamed..." if title is None else title, 'channels': slack['channel_id'] } try: r = requests.post("https://slack.com/api/files.upload", data=data, files={'file': file_content}) res = json.loads(r.content) if not res['ok']: return False if bot_callback is not None: r = requests.post( slack['webhook_url'], data={'payload': json.dumps({'text': bot_callback})}) print r.content return True except Exception as e: print e, type(e) return False
def __init__(self): print "xmpp client..." from conf import MONITOR_ROOT self.log_file = os.path.join(MONITOR_ROOT, "xmpp.log.txt") self.pid_file = os.path.join(MONITOR_ROOT, "xmpp.pid.txt") self.cred = getSecrets('xmpp')
def __on_message(self, msg): if msg['from'].jid.split('/')[0] != self.cred['bot_jid']: print "message not from our bot" return h = HTMLParser() try: cmd = json.loads(h.unescape(msg['body'])) except Exception as e: print "could not find json in message %s" % msg['body'] print e, type(e) return print cmd if len(cmd.keys()) != 2 or 'task_path' not in cmd.keys(): print "msg invalid (%s)" % msg['body'] return if cmd['task_path'] == "Documents.evaluate_document.evaluateDocument": if 'file_name' not in cmd.keys(): print "no file specified" return file_path = os.path.join(getSecrets('dropbox_root'), cmd['file_name']) while not os.path.exists(file_path): print "file not in dropbox yet" sleep(5) with settings(warn_only=True): local("cp %s %s" % (file_path, ANNEX_DIR)) local("rm %s" % file_path) whoami = local("whoami", capture=True) THIS_DIR = os.getcwd() annex_cmd = [ ".git/hooks/uv-post-netcat", "\"%s\"" % cmd['file_name'], "--importer_source=xmpp", "--imported_by=%s" % whoami ] os.chdir(ANNEX_DIR) with settings(warn_only=True): local(" ".join(annex_cmd)) os.chdir(THIS_DIR) else: req = "http://localhost:%d/reindex/?_id=%s&task_path=%s" % (API_PORT, cmd['doc_id'], cmd['task_path']) print req try: r = requests.get(req) except Exception as e: print "Could not do request" print e, type(e)
def __init__(self, func, args=None, op_dir=None): self.func = func if args is not None: self.args = args else: self.args = {} self.output = None self.error = None USE_SSH = getSecrets('server_force_ssh') if USE_SSH is None: if SERVER_HOST in ["127.0.0.1", "localhost"]: USE_SSH = False else: USE_SSH = True if USE_SSH: uv_user = getSecrets("server_user") hostname = getSecrets("server_host") port = getSecrets("annex_remote_port") port_prefix = "" if port is not None and port != 22: port_prefix += ":%d" % port self.args.update({ 'hosts' : ["%s@%s%s" % (uv_user, hostname, port_prefix)] }) if op_dir is not None: self.return_dir = os.getcwd() os.chdir(op_dir) threading.Thread.__init__(self) self.start()
def __init__(self): self.usable = True try: self.config = getSecrets('foxydoxxing_client')['gmail'] except Exception as e: if DEBUG: print e self.usable = False return try: with open(self.config['last_update'], 'rb') as L: self.last_update = float(L.read().strip()) except Exception as e: print e self.last_update = 0 if DEBUG: print "Last Intake: %d" % self.last_update # start up data service from oauth2client.file import Storage try: credentials = Storage(self.config['auth_storage']).get() except Exception as e: if DEBUG: print "NO CREDENTIALS YET." print e, type(e) self.usable = False return import httplib2 from apiclient.discovery import build http = httplib2.Http() http = credentials.authorize(http) try: self.service = build('gmail', 'v1', http=http) except Exception as e: if DEBUG: print "COULD NOT CREATE SERVICE:" print e, type(e) self.usable = False
def __init__(self, mode, tag=None): self.mime_types = copy.deepcopy(MIME_TYPES) self.mime_type_map = copy.deepcopy(MIME_TYPE_MAP) self.config = getSecrets(key="repo") if tag is not None: self.tag = tag try: with open(self.config['absorbed_log'], 'rb') as log: self.absorbed_log = json.loads(log.read()) except: self.absorbed_log = { 'sources': 0, 'submissions': 0 } self.mode = mode self.last_update_for_mode = self.absorbed_log[mode] self.usable = True
def share(self, fileId, email=None): if not hasattr(self, "service"): return None if email is None: email = getSecrets('annex_admin_email') if email is None: return None body = { 'role' : "writer", 'value' : email, 'type' : "user" } try: return self.service.permissions().insert(fileId=fileId, body=body).execute() except errors.HttpError as e: if DEBUG: print e return None
def __init__(self): self.usable = True api_config = {} try: self.config = getSecrets('foxydoxxing_client')['twitter'] for k in ['access_token_key', 'access_token_secret', 'consumer_secret', 'consumer_key']: if k not in self.config.keys(): self.usable = False break api_config[k] = self.config[k] except Exception as e: if DEBUG: print e self.usable = False if not self.usable: return self.service = twitter.Api(**api_config)
def __init__(self): if DEBUG: print "ANNEX CLIENT online" # get the conf settings try: self.hostname = getSecrets('server_host') except KeyError as e: pass try: self.port = getSecrets('server_port') except KeyError as e: pass try: self.user = getSecrets('server_user') except KeyError as e: pass try: self.remote_path = getSecrets('annex_remote') except KeyError as e: pass if getSecrets('auth_storage') is None: return from apiclient import errors from apiclient.discovery import build credentials = None try: from oauth2client.file import Storage credentials = Storage(getSecrets('auth_storage')).get() except KeyError as e: if DEBUG: print "NO AUTH YET!" if credentials is not None: import httplib2 http = httplib2.Http() http = credentials.authorize(http) self.service = build('drive', 'v2', http=http)
def netcat(file, save_as=None, alias=None, for_local_use_only=False, importer_source=None): whoami = "unknown" if importer_source is None or importer_source not in IMPORTER_SOURCES: print "NO IMPORTER SOURCE?" return None this_dir = os.getcwd() op_dir = this_dir with settings(warn_only=True): whoami = local("whoami", capture=True) res = [] if type(file) is str: if save_as is None: save_as = os.path.basename(file) else: if save_as is None: save_as = "uv_document_%d" % time() cmd_flags = [ "--importer_source=%s" % importer_source, "--imported_by=%s" % whoami ] if alias is not None: cmd_flags.append("--uv_file_alias=\"%s\"" % alias) if for_local_use_only: cmd_flags.append("--uv_local_only=True") cmd = [ ".git/hooks/uv-post-netcat \"%s\" %s" % (save_as, " ".join(cmd_flags)) ] if DEBUG: print "ATTEMPTING NETCAT:\nfile:%s" % save_as USE_SSH = getSecrets('server_force_ssh') print "USE SSH? %s" % USE_SSH if USE_SSH is None: if SERVER_HOST in ["127.0.0.1", "localhost"]: USE_SSH = False else: USE_SSH = True if USE_SSH: env.key_filename = [getSecrets('ssh_key_pub').replace(".pub", "")] with settings(warn_only=True): put_file = os.path.join(getSecrets('annex_remote'), save_as) put_cmd = put(file, put_file) if DEBUG: print put_cmd if len(put_cmd) == 0 or put_cmd[0] != put_file: return None res.append(put_cmd) use_hostname = env.host_string.split(":")[0] cmd = ["ssh -f -p %d -i %s %s -o IdentitiesOnly=yes 'cd %s && source ~/.bash_profile && %s'" % ( getSecrets('annex_remote_port'), env.key_filename[0], use_hostname, getSecrets('annex_remote'), c) for c in cmd] else: if type(file) in [str, unicode]: with settings(warn_only=True): res.append(local("mv %s %s" % (os.path.join(getSecrets('annex_local'), save_as), os.path.join(getSecrets('annex_remote'), save_as)), capture=True)) else: with open(os.path.join(getSecrets('annex_remote'), save_as), 'wb+') as F: F.write(file.getvalue()) file.close() op_dir = getSecrets('annex_remote') os.chdir(op_dir) if DEBUG: print "\n\nNOW LAUNCHING NETCAT COMMANDS (%d)" % len(cmd) print "in dir %s" % os.getcwd() for c in cmd: if DEBUG: print "\n%s\n" % c res.append(local(c, capture=True)) os.chdir(this_dir) if DEBUG: print "*************\n\n%s\n\n*************" % res print "returning from netcat." return res
def doIntake(task): task_tag = "INTAKE" print "\n\n************** %s [START] ******************\n" % task_tag task.setStatus(302) next_mode = None if not hasattr(task, "mode"): mode = "sources" next_mode = "submissions" else: mode = task.mode del task.mode if mode not in ["sources", "submissions"]: print "No acceptable mode" print "\n\n************** %s [ERROR] ******************\n" % task_tag task.fail() return repo = None try: from conf import getSecrets repo = getSecrets("repo") except Exception as e: print "ERROR GETTING REPO: %s" % e pass if repo is None: print "No repos to query." print "\n\n************** %s [ERROR] ******************\n" % task_tag task.fail() return client = None if repo["source"] == "google_drive": from lib.Worker.Models.ic_google_drive_client import InformaCamDriveClient try: client = InformaCamDriveClient(mode=mode) except Exception as e: print "ERROR Creating client:\n%s" % e print "\n\n************** %s [ERROR] ******************\n" % task_tag task.fail() return elif repo["source"] == "globaleaks": from lib.Worker.Models.ic_globaleaks_client import InformaCamGlobaleaksClient client = InformaCamGlobaleaksClient(mode=mode) elif repo["source"] == "s3bucket": from lib.Worker.Models.ic_s3_client import InformaCamS3Client client = InformaCamS3Client(mode=mode) if client is None or not client.usable: print "Client invalid." print "\n\n************** %s [ERROR] ******************\n" % task_tag task.fail() return from time import sleep try: for asset in client.listAssets(omit_absorbed=True): mime_type = client.getAssetMimeType(asset) if not mime_type in client.mime_types.itervalues(): continue if client.download(asset) is not None: client.absorb(asset) sleep(5) except TypeError as e: print e if next_mode is not None: task.mode = next_mode doIntake(task) else: task.finish() print "\n\n************** %s [END] ******************\n" % task_tag
def netcat(file, save_as=None, alias=None, for_local_use_only=False, importer_source=None): whoami = "unknown" if importer_source is None or importer_source not in IMPORTER_SOURCES: print "NO IMPORTER SOURCE?" return None this_dir = os.getcwd() op_dir = this_dir with settings(warn_only=True): whoami = local("whoami", capture=True) res = [] if type(file) is str: if save_as is None: save_as = os.path.basename(file) else: if save_as is None: save_as = "uv_document_%d" % time() cmd_flags = [ "--importer_source=%s" % importer_source, "--imported_by=%s" % whoami ] if alias is not None: cmd_flags.append("--uv_file_alias=\"%s\"" % alias) if for_local_use_only: cmd_flags.append("--uv_local_only=True") cmd = [ ".git/hooks/uv-post-netcat \"%s\" %s" % (save_as, " ".join(cmd_flags)) ] if DEBUG: print "ATTEMPTING NETCAT:\nfile:%s" % save_as USE_SSH = getSecrets('server_force_ssh') print "USE SSH? %s" % USE_SSH if USE_SSH is None: if SERVER_HOST in ["127.0.0.1", "localhost"]: USE_SSH = False else: USE_SSH = True if USE_SSH: env.key_filename = [getSecrets('ssh_key_pub').replace(".pub", "")] with settings(warn_only=True): put_file = os.path.join(getSecrets('annex_remote'), save_as) put_cmd = put(file, put_file) if DEBUG: print put_cmd if len(put_cmd) == 0 or put_cmd[0] != put_file: return None res.append(put_cmd) use_hostname = env.host_string.split(":")[0] cmd = [ "ssh -f -p %d -i %s %s -o IdentitiesOnly=yes 'cd %s && source ~/.bash_profile && %s'" % (getSecrets('annex_remote_port'), env.key_filename[0], use_hostname, getSecrets('annex_remote'), c) for c in cmd ] else: if type(file) in [str, unicode]: with settings(warn_only=True): res.append( local("mv %s %s" % (os.path.join(getSecrets('annex_local'), save_as), os.path.join(getSecrets('annex_remote'), save_as)), capture=True)) else: with open(os.path.join(getSecrets('annex_remote'), save_as), 'wb+') as F: F.write(file.getvalue()) file.close() op_dir = getSecrets('annex_remote') os.chdir(op_dir) if DEBUG: print "\n\nNOW LAUNCHING NETCAT COMMANDS (%d)" % len(cmd) print "in dir %s" % os.getcwd() for c in cmd: if DEBUG: print "\n%s\n" % c res.append(local(c, capture=True)) os.chdir(this_dir) if DEBUG: print "*************\n\n%s\n\n*************" % res print "returning from netcat." return res
def decrypt(uv_task): task_tag = "DECRYPTING" print "\n\n************** %s [START] ******************\n" % task_tag print "decrypting pgp blob for %s" % uv_task.doc_id uv_task.setStatus(302) from lib.Worker.Models.uv_document import UnveillanceDocument media = UnveillanceDocument(_id=uv_task.doc_id) if media is None: print "DOC IS NONE" print "\n\n************** %s [ERROR] ******************\n" % task_tag uv_task.fail() return if not media.getFile(uv_task.pgp_file): print "NO PGP FILE" print "\n\n************** %s [ERROR] ******************\n" % task_tag uv_task.fail() return from conf import getSecrets gpg_pwd = getSecrets("gpg_pwd") if gpg_pwd is None: err_msg = "NO PASSPHRASE TO DECRYPT" print err_msg print "\n\n************** %s [ERROR] ******************\n" % task_tag uv_task.fail(message=err_msg) return gpg_dir = getSecrets("gpg_dir") # save as task.pgp_file.decrypted or whatever import os from fabric.api import local, settings from fabric.context_managers import hide from conf import ANNEX_DIR, DEBUG if not hasattr(uv_task, "save_as"): save_as = "%s.decrypted" % uv_task.pgp_file else: save_as = uv_task.save_as print "\n\n************** %s [INFO] ******************\n" % task_tag print "SAVING DECRYPTED ASSET TO %s IF SUCCESSFUL" % save_as with settings(hide("everything"), warn_only=True): d_cmd = "gpg --yes --no-tty --homedir=%s --passphrase %s --output %s --decrypt %s" % ( gpg_dir, gpg_pwd, os.path.join(ANNEX_DIR, save_as), os.path.join(ANNEX_DIR, uv_task.pgp_file), ) decrypted = local(d_cmd) print decrypted.return_code del gpg_pwd if decrypted.return_code == 2: err_msg = "could not successfully decrypt %s" % uv_task.pgp_file print err_msg print "\n\n************** %s [ERROR] ******************\n" % task_tag uv_task.fail(status=412, message=err_msg) return media.addCompletedTask(uv_task.task_path) if uv_task.get_next() is None: # route according to mime type # get mime type of decrypted from vars import MIME_TYPE_TASKS from lib.Worker.Utils.funcs import getFileType mime_type = getFileType(os.path.join(ANNEX_DIR, save_as)) # usable: json (a j3m), zip (a source or a log->batch) if mime_type in MIME_TYPE_TASKS.keys(): print "mime type (%s) usable..." % mime_type try: uv_task.put_next(MIME_TYPE_TASKS[mime_type]) except Exception as e: print e uv_task.routeNext(inflate={"file_name": save_as}) uv_task.finish() print "\n\n************** %s [END] ******************\n" % task_tag