Ejemplo n.º 1
0
	def get(self, submission_id):
		"""Returns the specified Submission.
		
		required:
			_id
		"""
		res = ServerResponse()
		
		if passesParameterFilter(submission_id):
			submission = ICSubmission(_id = submission_id)
			
			if not hasattr(submission, 'invalid'):
				res.data = submission.emit()
				res.result = 200
			else:
				res.reason = submission.invalid
				
			if len(self.request.query) > 3:
				with_query = submission.search(parseRequest(self.request.query))
				res.data['refined_search'] = {}
				
				if hasattr(with_query, 'annotations'):
					res.data['refined_search']['annotations'] = with_query.annotations

		self.write(res.emit())
Ejemplo n.º 2
0
def reindex():
    from vars import mime_type_map
    from InformaCamUtils.elasticsearch import Elasticsearch

    elasticsearch = Elasticsearch()
    elasticsearch.createIndex(reindex=True)

    for mode in ["sources", "submissions"]:
        for root_, dirs_, files_ in os.walk(os.path.join(assets_root, mode)):
            for dir_ in dirs_:
                for root, dirs, files in os.walk(
                        os.path.join(assets_root, mode, dir_)):
                    data = {
                        '_id': dir_,
                        'asset_path': os.path.join(assets_root, mode, dir_)
                    }

                    for file in files:
                        if mode == "submissions":
                            if re.match(r'^(low_|med_|thumb_|high_)', file):
                                continue
                            else:
                                comps = file.split(".")
                                if len(comps) == 2 and re.match(
                                        r'(mkv|jpg)', comps[1]):
                                    data['file_name'] = file
                                    for mt, ext in mime_type_map.iteritems():
                                        if ext == comps[1]:
                                            data['mime_type'] = mt
                                            break
                                    break
                        elif mode == "sources":
                            if re.match(r'^(credentials|publicKey|baseImage_)',
                                        file):
                                continue
                            else:
                                data['file_name'] = file
                                break

                    print data

                    try:
                        if mode == "submissions":
                            submission = Submission(inflate=data, reindex=True)
                            print submission.emit()
                        elif mode == "sources":
                            source = Source(inflate=data, reindex=True)
                    except exceptions.ConnectionError as e:
                        print e
                        return
                    except AttributeError as e:
                        continue
Ejemplo n.º 3
0
	def get(self, _id):
		res = Res()
		
		if passesParameterFilter(_id):
			submission = ICSubmission(_id=_id)
			
			if not hasattr(submission, "invalid"):
				res.data = submission.emit()
				res.result = 200
			else:
				res.reason = submission.invalid
				
		self.finish(res.emit())
Ejemplo n.º 4
0
	def get(self, _id):
		res = Res()
		
		if passesParameterFilter(_id):
			submission = ICSubmission(_id=_id)
			
			if not hasattr(submission, "invalid"):
				res.data = submission.emit()
				res.result = 200
			else:
				res.reason = submission.invalid
				
		self.finish(res.emit())
Ejemplo n.º 5
0
def reindex():
	from vars import mime_type_map
	from InformaCamUtils.elasticsearch import Elasticsearch
	
	elasticsearch = Elasticsearch()
	elasticsearch.createIndex(reindex=True)
	
	for mode in ["sources", "submissions"]:
		for root_, dirs_, files_ in os.walk(os.path.join(assets_root, mode)):
			for dir_ in dirs_:
				for root, dirs, files in os.walk(
					os.path.join(assets_root, mode, dir_)):
					data = {
						'_id' : dir_,
						'asset_path' : os.path.join(assets_root, mode, dir_)
					}
					
					for file in files:
						if mode == "submissions":
							if re.match(r'^(low_|med_|thumb_|high_)', file):
								continue
							else:
								comps = file.split(".")
								if len(comps) == 2 and re.match(r'(mkv|jpg)', comps[1]):
									data['file_name'] = file
									for mt, ext in mime_type_map.iteritems():
										if ext == comps[1]:
											data['mime_type'] = mt
											break
									break
						elif mode == "sources":
							if re.match(r'^(credentials|publicKey|baseImage_)', file):
								continue
							else:
								data['file_name'] = file
								break
					
					print data

					try:
						if mode == "submissions":
							submission = Submission(inflate=data, reindex=True)
							print submission.emit()
						elif mode == "sources":
							source = Source(inflate=data, reindex=True)
					except exceptions.ConnectionError as e:
						print e
						sys.exit()
					except AttributeError as e:
						continue
Ejemplo n.º 6
0
	def post(self):
		"""Add a new Submission to the database.
		
		Only the server may utilize this method.
		
		required parameters in post:
			_id
			package_name
			package_content
		"""
		
		res = ServerResponse()		
		params = parseRequest(self.request.body)

		if params is not None:
			submission = ICSubmission(inflate={'_id' : params['_id']})
			if submission.addFile(params['package_name'], params['package_content']):
				if submission.importAssets(params['package_name']):
					submission.setMimeType(params['mime_type'])
					
					res.result = 200
					res.data = submission.emit()
				else:
					submission.invalidate(
						invalidate['codes']['asset_non_existent'],
						invalidate['reasons']['asset_non_existent']
					)
					res.reason = submission.invalid
		
		self.write(res.emit())
Ejemplo n.º 7
0
	def post(self, _id):
		res = Res()
		
		if passesParameterFilter(_id):
			submission = ICSubmission(_id=_id)
			
			if not hasattr(submission, "invalid"):
				for k,v in parseRequest(self.request.body).iteritems():
					if k not in submission.locked_fields:
						setattr(source, k, v)
				
				if submission.save():
					res.data = submission.emit()
					res.result = 200
			
		self.finish(res.emit())
Ejemplo n.º 8
0
	def post(self, _id):
		res = Res()
		
		if passesParameterFilter(_id):
			submission = ICSubmission(_id=_id)
			
			if not hasattr(submission, "invalid"):
				for k,v in parseRequest(self.request.body).iteritems():
					if k not in submission.locked_fields:
						setattr(source, k, v)
				
				if submission.save():
					res.data = submission.emit()
					res.result = 200
			
		self.finish(res.emit())
Ejemplo n.º 9
0
    def __init__(self, submission):
        from InformaCamModels.submission import Submission
        from InformaCamModels.collection import Collection
        from InformaCamModels.j3m import J3M

        # unzip
        self.asset_path = submission.asset_path.replace(
            "/submissions/", "/collections/")
        try:
            os.makedirs(self.asset_path)
        except OSError as e:
            print e

        print submission.emit()
        unzip = ShellThreader([
            "unzip",
            "%s/%s" % (submission.asset_path, submission.file_name), "-d",
            self.asset_path
        ])
        unzip.start()
        unzip.join()
        print unzip.output

        assets = []
        media = []
        sensor_captures = []
        j3m_verified = False
        j3m = None

        rx = r'\s*inflating: (.*)'
        for line in unzip.output:
            #for line in unzip_output:
            asset_match = re.findall(rx, line)
            if len(asset_match) == 1:
                assets.append(asset_match[0].strip())

        r_j3m = r'.*/log.j3m'
        r_caches = r'.*/informaCaches/.*'

        for asset in assets:
            j3m_match = re.match(r_j3m, asset)
            if j3m_match is not None:
                print "J3M: %s" % asset
                j3m_verified = self.validateManifest()
                j3m = J3M(path_to_j3m="%s.manifest" % asset)

            cache_match = re.match(r_caches, asset)
            if cache_match is not None:
                print "ADD CACHE: %s" % asset
                sensor_captures.append(asset)

            else:
                mime_type = getMimeTypeFromFile(asset)
                print mime_type
                if mime_type in [mime_types['image'], mime_types['video']]:
                    print "HERE IS A NEW SUB:%s" % asset

                    f = open(asset, 'rb')
                    data = {
                        '_id': hashString(asset),
                        'file_name': os.path.basename(asset),
                        'mime_type': mime_type,
                        'package_content': b64encode(f.read()).rstrip("="),
                        'sync_source': submission.sync_source
                    }
                    f.close()

                    if data['file_name'][
                            -4:] != ".%s" % mime_type_map[mime_type]:
                        data['file_name'] = "%s.%s" % (
                            data['file_name'], mime_type_map[mime_type])

                    try:
                        new_submission = Submission(inflate=data)
                        media.append(new_submission._id)
                    except exceptions.ConnectionError as e:
                        print e
                        return

        inflate = {'_id': submission._id, 'asset_path': self.asset_path}

        if len(media) > 0:
            inflate['attached_media'] = media

        if len(sensor_captures) > 0:
            inflate['sensor_captures'] = sensor_captures

        if j3m is not None:
            inflate['j3m_id'] = j3m._id
            inflate['dateCreated'] = j3m.genealogy['dateCreated']

        collection = Collection(inflate=inflate)
        print "NEW COLLECTION: %s" % collection._id

        # delete self as submission
        submission.delete()
Ejemplo n.º 10
0
def watch(only_sources=False, only_submissions=False, only_imports=False):
    """For each subscribed repository, this class sends new media to our Data API.
	
	"""
    clients = []
    mode = None

    if only_submissions:
        mode = "submissions"
    elif only_sources:
        mode = "sources"

    print "running watch... (mode=%s)" % mode

    for sync_type in sync:
        if sync_type == "google_drive":
            if not only_imports:
                from InformaCamData.drive_client import DriveClient
                clients.append(DriveClient(mode=mode))
        elif sync_type == "globaleaks":
            if not only_imports:
                from InformaCamData.globaleaks_client import GlobaleaksClient
                clients.append(GlobaleaksClient(mode=mode))
        elif sync_type == "import":
            from InformaCamData.import_client import ImportClient
            clients.append(ImportClient(mode=mode))

    for client in clients:
        if not client.usable:
            print "client not usable"
            continue

        for asset in client.listAssets(omit_absorbed=True):
            mime_type = client.getAssetMimeType(asset)
            if not mime_type in client.mime_types.itervalues():
                continue

            if mime_type == client.mime_types['zip']:
                if only_submissions:
                    continue

                data = {
                    '_id':
                    client.getFileNameHash(asset),
                    'file_name':
                    client.getFileName(asset),
                    'package_content':
                    b64encode(client.pullFile(asset)).rstrip("="),
                    'sync_source':
                    sync_type
                }
                print "%s is a source" % data['file_name']

                try:
                    source = Source(inflate=data)
                except exceptions.ConnectionError as e:
                    print e
                    return

                if hasattr(source, "invalid"):
                    print source.invalid
                    continue

            else:
                if only_sources:
                    continue

                data = {
                    '_id':
                    client.getFileNameHash(asset),
                    'file_name':
                    client.getFileName(asset),
                    'mime_type':
                    mime_type,
                    'package_content':
                    b64encode(client.pullFile(asset)).rstrip("="),
                    'sync_source':
                    sync_type
                }
                print "%s is a submission" % data['file_name']

                file_name_segments = getBespokeFileExtension(data['file_name'])
                if file_name_segments is None:
                    print "could not map file extension for %s" % data[
                        'file_name']
                    file_name_segments = ['']

                if file_name_segments[-1] != client.mime_type_map[mime_type]:
                    data['file_name'] = "%s.%s" % (
                        data['file_name'], client.mime_type_map[mime_type])

                try:
                    submission = Submission(inflate=data)
                except exceptions.ConnectionError as e:
                    print e
                    return

                if hasattr(submission, "invalid"):
                    print submission.invalid
                    continue

            client.absorb(asset)
            client.lockFile(asset)
        client.updateLog()