def send(self, request, stream=None, timeout=None, verify=None, cert=None, proxies=None):
pathname = url_to_path(request.url)
resp = Response()
resp.status_code = 200
resp.url = request.url
try:
stats = lstat(pathname)
except (IOError, OSError) as exc:
resp.status_code = 404
message = {
"error": "file does not exist",
"path": pathname,
"exception": repr(exc),
}
fh = SpooledTemporaryFile()
fh.write(ensure_binary(json.dumps(message)))
fh.seek(0)
resp.raw = fh
resp.close = resp.raw.close
else:
modified = formatdate(stats.st_mtime, usegmt=True)
content_type = guess_type(pathname)[0] or "text/plain"
resp.headers = CaseInsensitiveDict({
"Content-Type": content_type,
"Content-Length": stats.st_size,
"Last-Modified": modified,
})
resp.raw = open(pathname, "rb")
resp.close = resp.raw.close
return resp
def archive_to_repo(archive_path, repo, archive_type="tar"):
"""Downloads a archive from the specified path,
extracts it into the repo's directory, commits
any changes from the previous version and pushes it
to github!!!!"""
# Download the tarball and stick it in a tempfile
r = requests.get(archive_path)
tmp = SpooledTemporaryFile()
tmp.write(r.content)
tmp.seek(0)
# Open the tempfile contents as an actual tarball
if archive_type == "tar":
archive = tarfile.open(fileobj=tmp)
elif archive_type == "zip":
archive = zipfile.open(tmp)
else:
raise ValueError("Unrecognized Archive Type")
# Clear working files
clear_working_dir(repo.working_dir)
# Extract to the repo path
archive.extract(repo.working_dir)
# Add and commit everything!
try:
repo.git.add(".", A=True)
repo.git.commit(m="New archive version")
except:
pass # May be that there was nothing new to commit
# Cleanup, outta here!
archive.close()
tmp.close()
def sponge(
*,
filename: str,
append: bool = False,
tee: bool = False,
max_memory_size: int,
debug: bool,
) -> None:
"""soak up all input from stdin and write it to"""
from tempfile import SpooledTemporaryFile
with contextlib.ExitStack() as s:
tmpio = SpooledTemporaryFile(max_size=max_memory_size,
mode="w+",
prefix="sponge.")
for line in sys.stdin:
tmpio.write(line)
has_file = bool(tmpio.name is not None)
if has_file:
logger.info("sponge: using temporary file (%s)", tmpio.name)
tmpio = s.enter_context(tmpio)
mode = "a" if append else "w"
with open(filename, mode) as wf:
tmpio.seek(0)
if tee:
for line in tmpio:
wf.write(line)
sys.stdout.write(line)
else:
for line in tmpio:
wf.write(line)
def gen_data(location=None, **kwargs):
"""Fetches realtime data and generates records"""
url = '%s/%s' % (kwargs['BASE_URL'], location)
r = requests.get(url)
f = SpooledTemporaryFile() # wrap to access `fileno`
f.write(r.content)
return io.read_xls(r., sanitize=True, encoding=r.encoding)
def get_record(model_id):
"""Get the full record of a stored model. This includes the cimobj and the xml files.
:param model_id: Model id fs
:type model_id: int
:rtype: record
"""
try:
model = redis_connection.get(model_id).decode("utf-8")
cimpy_data = redis_connection.get(str(model_id) + "_cimpy").decode("utf-8")
files_len = int(
redis_connection.get(str(model_id) + "_files_len").decode("utf-8")
)
except AttributeError:
raise KeyError
files = []
for index in range(files_len):
data_addr = str(model_id) + "_file_" + str(index)
data = redis_connection.get(data_addr)
newfile = SpooledTemporaryFile()
newfile.write(data)
files.append(newfile)
return record(model, cimpy_data, files)
def request(self, url, params={}, headers={}):
"""
Retrieve SDMX messages.
If needed, override in subclasses to support other data providers.
:param url: The URL of the message.
:type url: str
:return: the xml data as file-like object
"""
# Generate current config. Merge in any given headers
cur_config = self.config.copy()
if 'headers' in cur_config:
cur_config['headers'] = cur_config['headers'].copy()
cur_config['headers'].update(headers)
else:
cur_config['headers'] = headers
with closing(requests.get(url, params=params, **cur_config)) as response:
if response.status_code == requests.codes.OK:
source = STF(max_size=self.max_size)
for c in response.iter_content(chunk_size=1000000):
source.write(c)
else:
source = None
code = int(response.status_code)
if 400 <= code <= 499:
raise response.raise_for_status()
return source, response.url, response.headers, code
def send(self, request, stream=None, timeout=None, verify=None, cert=None, proxies=None):
pathname = url_to_path(request.url)
resp = Response()
resp.status_code = 200
resp.url = request.url
try:
stats = stat(pathname)
except (IOError, OSError) as exc:
resp.status_code = 404
message = {
"error": "file does not exist",
"path": pathname,
"exception": repr(exc),
}
fh = SpooledTemporaryFile()
fh.write(ensure_binary(json.dumps(message)))
fh.seek(0)
resp.raw = fh
resp.close = resp.raw.close
else:
modified = formatdate(stats.st_mtime, usegmt=True)
content_type = guess_type(pathname)[0] or "text/plain"
resp.headers = CaseInsensitiveDict({
"Content-Type": content_type,
"Content-Length": stats.st_size,
"Last-Modified": modified,
})
resp.raw = open(pathname, "rb")
resp.close = resp.raw.close
return resp
def request(self, url, params={}, headers={}):
"""
Retrieve SDMX messages.
If needed, override in subclasses to support other data providers.
:param url: The URL of the message.
:type url: str
:return: the xml data as file-like object
"""
# Generate current config. Merge in any given headers
cur_config = self.config.copy()
if 'headers' in cur_config:
cur_config['headers'] = cur_config['headers'].copy()
cur_config['headers'].update(headers)
else:
cur_config['headers'] = headers
with closing(requests.get(url, params=params,
**cur_config)) as response:
if response.status_code == requests.codes.OK:
source = STF(max_size=self.max_size)
for c in response.iter_content(chunk_size=1000000):
source.write(c)
else:
source = None
code = int(response.status_code)
if 400 <= code <= 499:
raise response.raise_for_status()
return source, response.url, response.headers, code
def _get_attachment(part):
incr_if_enabled('email_with_attachment', 1)
fn = part.get_filename()
ct = part.get_content_type()
payload = part.get_payload(decode=True)
payload_size = len(payload)
if fn:
extension = os.path.splitext(fn)[1]
else:
extension = mimetypes.guess_extension(ct)
tag_type = 'attachment'
attachment_extension_tag = generate_tag(tag_type, extension)
attachment_content_type_tag = generate_tag(tag_type, ct)
histogram_if_enabled(
'attachment.size',
payload_size,
[attachment_extension_tag, attachment_content_type_tag]
)
attachment = SpooledTemporaryFile(
max_size=150*1000, # 150KB max from SES
suffix=extension,
prefix=os.path.splitext(fn)[0]
)
attachment.write(payload)
return fn, attachment
def fetch_data(config):
"""Fetches realtime data and generates records"""
ckan = CKAN(config['ENDPOINT'], apikey=config['API_KEY'])
# r = ckan.fetch_resource(config['RID']) # if using ckanutils
resource = ckan.action.resource_show(id=config['RID'])
url = resource.get('perma_link') or resource.get('url')
r = requests.get(url, stream=True)
if any('403' in h.headers.get('x-ckan-error', '') for h in r.history):
raise NotAuthorized('Access to fetch resource %s was denied.' %
config['RID'])
try:
ext = splitext(url)[1].split('.')[1]
except IndexError:
ext = cv.ctype2ext(r.headers['Content-Type'])
if ext == 'csv':
records = io.read_csv(r.raw, sanitize=True, encoding=r.encoding)
elif ext in {'xls', 'xlsx'}:
r = requests.get(url)
f = SpooledTemporaryFile()
f.write(r.content)
records = io.read_xls(f, sanitize=True, encoding=r.encoding)
else:
msg = 'Filetype `%s` unsupported.'
msg += 'Please view tabutils.io documentation for assistance.'
raise TypeError(msg)
constraints = [('adm0_name', 'a'), ('mp_month', '3'), ('mp_year', '2015')]
filterer = lambda x: all(x[k].lower().startswith(v)
for k, v in constraints)
return it.ifilter(filterer, records)
def download_slack_file(file_id: str,
slack_client) -> Tuple[str, SpooledTemporaryFile]:
slack_file = slack_client.api_call('files.info', file=file_id)
url = slack_file.get('file').get('url_private')
file_name = os.path.basename(urlparse(url).path)
file_type = guess_type(url)[0]
max_chunk = int(1e6) # 1MiB
tmp = SpooledTemporaryFile(suffix=file_name,
mode='w+b',
max_size=max_chunk)
headers = {
'user-agent': 'github.com/austinpray/kaori',
'Authorization': f'Bearer {slack_client.token}',
}
with requests.get(url, headers=headers, stream=True) as resp:
if resp.status_code != 200:
raise RuntimeError('non-200 on image')
content_type = resp.headers['content-type']
if file_type not in content_type:
raise RuntimeError(
f'wrong filetype {content_type}, expected {file_type}')
for chunk in resp.iter_content(chunk_size=max_chunk):
tmp.write(chunk)
tmp.seek(0)
return file_name, tmp
def push_index(self):
stream = SpooledTemporaryFile(max_size=20 * MB)
pointers = 0
stream.write(struct.pack(OFFSET_FMT, pointers))
self.indexes.append([
stream, pointers, self.block_size - self.pointer_size
])
def string2spool(input_string):
"""Takes a string as an argument and returns an open file handle with the
contents of the string"""
file_object=SpooledTemporaryFile()
file_object.write(input_string)
file_object.seek(0)
return file_object
def fetch_data(config):
"""Fetches realtime data and generates records"""
ckan = CKAN(config['ENDPOINT'], apikey=config['API_KEY'])
# r = ckan.fetch_resource(config['RID']) # if using ckanutils
resource = ckan.action.resource_show(id=config['RID'])
url = resource.get('perma_link') or resource.get('url')
r = requests.get(url, stream=True)
if any('403' in h.headers.get('x-ckan-error', '') for h in r.history):
raise NotAuthorized(
'Access to fetch resource %s was denied.' % config['RID'])
try:
ext = splitext(url)[1].split('.')[1]
except IndexError:
ext = cv.ctype2ext(r.headers['Content-Type'])
if ext == 'csv':
records = io.read_csv(r.raw, sanitize=True, encoding=r.encoding)
elif ext in {'xls', 'xlsx'}:
r = requests.get(url)
f = SpooledTemporaryFile()
f.write(r.content)
records = io.read_xls(f, sanitize=True, encoding=r.encoding)
else:
msg = 'Filetype `%s` unsupported.'
msg += 'Please view tabutils.io documentation for assistance.'
raise TypeError(msg)
constraints = [('adm0_name', 'a'), ('mp_month', '3'), ('mp_year', '2015')]
filterer = lambda x: all(x[k].lower().startswith(v) for k, v in constraints)
return it.ifilter(filterer, records)
class InputStream(object):
"""
FCGI_STDIN or FCGI_DATA stream.
Uses temporary file to store received data once max_mem bytes
have been received.
"""
def __init__(self, max_mem=1024):
self._file = SpooledTemporaryFile(max_mem)
self._eof_received = Event()
def feed(self, data):
if self._eof_received.is_set():
raise IOError('Feeding file beyond EOF mark')
if not data: # EOF mark
self._file.seek(0)
self._eof_received.set()
else:
self._file.write(data)
def __iter__(self):
self._eof_received.wait()
return iter(self._file)
def read(self, size=-1):
self._eof_received.wait()
return self._file.read(size)
def readlines(self, sizehint=0):
self._eof_received.wait()
return self._file.readlines(sizehint)
@property
def eof_received(self):
return self._eof_received.is_set()
def _open(self, name, mode = 'rb') -> File:
name = self._transform_name(name)
content = self.service.get_blob_content(self.container, name)
file = SpooledTemporaryFile()
file.write(content)
file.seek(0) # explicitly reset to allow reading from the beginning afterwards as-is
return File(file)
def _save_content(self, obj, content, parameters):
content.seek(0, os.SEEK_SET)
content_autoclose = SpooledTemporaryFile()
content_autoclose.write(content.read())
super(CustomS3Boto3Storage,
self)._save_content(obj, content_autoclose, parameters)
if not content_autoclose.closed:
content_autoclose.close()
def push_index(self):
stream = SpooledTemporaryFile(max_size=20 * MB)
pointers = 0
stream.write(struct.pack(OFFSET_FMT, pointers))
self.indexes.append(
[stream, pointers, self.block_size - self.pointer_size])
def decode_bytes(bytes):
temp = SpooledTemporaryFile()
string = ""
for byte in bytes:
string = string + chr(byte)
temp.write(string)
temp.seek(0) # work around a stupid python bug
return Decode(0, temp.read(), Decode64Bits)
def create_zipfile_storage(
opened_file: BufferedReader, filename: str,
) -> FileStorage:
spooled = SpooledTemporaryFile()
spooled.write(opened_file.read())
zip_file_storage = FileStorage(spooled)
zip_file_storage.filename = filename
opened_file.seek(0)
return zip_file_storage
def test_run_command_stdin(self):
connector = BaseCommandDBConnector()
stdin = SpooledTemporaryFile()
stdin.write(b'foo')
stdin.seek(0)
# Run
stdout, stderr = connector.run_command('cat', stdin=stdin)
self.assertEqual(stdout.read(), b'foo')
self.assertFalse(stderr.read())
def file_from_content(content):
f = content
if isinstance(content, cgi.FieldStorage):
f = content.file
elif isinstance(content, byte_string):
f = SpooledTemporaryFile(INMEMORY_FILESIZE)
f.write(content)
f.seek(0)
return f
def _http_get(self, path: str) -> SpooledTemporaryFile:
url = urllib.parse.urljoin(self.repo_url.rstrip("/") + "/", path)
logger.debug("Fetching %s", url)
response = self._session.get(url, headers=HEADERS)
buffer = SpooledTemporaryFile(max_size=100 * 1024 * 1024)
for chunk in response.iter_content(chunk_size=10 * 1024 * 1024):
buffer.write(chunk)
buffer.flush()
buffer.seek(0)
return buffer
def filter_file(filter, filename, membuffer=10485760):
tmp = SpooledTemporaryFile(max_size=membuffer)
with open(filename) as input:
for line in input:
if filter(line):
tmp.write(line)
tmp.seek(0)
with open(filename, "w") as output:
for line in tmp:
output.write(line)
def load_stix(stix):
# Just save the pain and load it if the first character is a <
if sys.version_info < (3, 5):
json_exception = ValueError
else:
json_exception = json.JSONDecodeError
if isinstance(stix, STIXPackage):
# Oh cool we're ok
# Who tried to load this? Honestly.
return stix
elif hasattr(stix, 'read'):
# It's a file!
# But somehow, sometimes, reading it returns a bytes stream and the loader dies on python 3.4.
# Luckily, STIXPackage.from_json (which is mixbox.Entity.from_json) will happily load a string.
# So we're going to play dirty.
data = stix.read()
if isinstance(data, bytes):
data = data.decode()
try:
# Try loading from JSON
stix_package = STIXPackage.from_json(data)
except json_exception:
# Ok then try loading from XML
# Loop zoop
stix.seek(0)
try:
stix_package = STIXPackage.from_xml(stix)
except Exception as ex:
# No joy. Quit.
raise STIXLoadError("Could not load stix file. {}".format(ex))
return stix_package
elif isinstance(stix, str):
# It's text, we'll need to use a temporary file
# Create a temporary file to load from
# Y'know I should probably give it a max size before jumping to disk
# idk, 10MB? Sounds reasonable.
f = SpooledTemporaryFile(max_size=10 * 1024)
# O I have idea for sneak
# Will be very sneak
# Write the (probably) XML to file
f.write(stix.encode("utf-8"))
# Reset the file so we can read from it
f.seek(0)
# AHA SNEAK DIDN'T EXPECT RECURSION DID YOU
return load_stix(f)
def run(size_in_gb):
create_creds_folder()
dummy = SpooledTemporaryFile()
for i in range(0, size_in_gb, 1):
dummy.write(RANDOM_DATA * 1024)
dummy.seek(0)
file = Dummy(dummy)
Drive().upload(file)
def pipestring_process(cmd_string, stdin_string=''):
"""Pipe a python string to standard input for cmd_string
>>> pipestring_process('grep 2', '1\\n2\\n3\\n')
(0, '2\\n', '')
"""
f=SpooledTemporaryFile()
f.write(stdin_string)
f.seek(0)
results=process(cmd_string, stdin=f)
f.close()
return results
def store_filelike(self, filelike):
copy = SpooledTemporaryFile() # boto3 now closes the file.
copy.write(filelike.read())
copy.seek(0)
details = {'ACL': self.acl}
if self.content_type:
details['ContentType'] = self.content_type
self._key.upload_fileobj(copy, ExtraArgs=details)
return self.url()
class BloomStringSet:
"""
BloomSet implements set membership in a fixed memory footprint,
but can reproduce the keys by streaming them to temporary files.
As it's based on a bloom filter, there is a risk of false positives,
which would cause some missing keyue, however the likelihood
of such is controllable through the parameters. Temporary files
are only created after they reach 5MB, otherwise stay in memory.
cardinality: the estimated maximum number of unique elements.
As one goes beyond this number, risk of collision
increases, but sloooooowly.
error_rate: the false positive rate you are comfortable with when
the cardinality number is reached.
"""
def __init__(self, cardinality=10**6, error_rate=10**-9):
self.bloom = BloomFilter(cardinality, error_rate)
self.file = SpooledTemporaryFile(max_size=(2**20) * 5, mode='w')
self.closed = False
def add(self, key):
if self.closed:
raise Exception("Cannot add new element after attempting to read")
if type(key) is not str:
raise Exception("Can only use string keys for now")
if key in self.bloom:
return False
self.bloom.add(key)
self.file.write(key + "\n")
def __contains__(self, key):
return key in self.bloom
def __iter__(self):
self.closed = True
self.file.seek(0)
return self
def __next__(self):
line = self.file.readline()
if line == '':
self.file.close()
raise StopIteration
return line.strip()
def __del__(self):
self.file.close()
def retrieve_shard_file(self, url, shard_index):
farmer_tries = 0
self.__logger.debug('Downloading shard at index %s from farmer: %s',
shard_index, url)
tries_download_from_same_farmer = 0
while MAX_RETRIES_DOWNLOAD_FROM_SAME_FARMER > \
tries_download_from_same_farmer:
tries_download_from_same_farmer += 1
farmer_tries += 1
try:
# data is spooled in memory until the file size exceeds max_size
shard = SpooledTemporaryFile(self.max_spooled, 'wb')
# Request the shard
r = requests.get(url, stream=True, timeout=self.client.timeout)
if r.status_code != 200 and r.status_code != 304:
raise StorjFarmerError()
# Write the file
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
shard.write(chunk)
# Everything ok
# move file read pointer at beginning
shard.seek(0)
return shard
except StorjFarmerError as e:
self.__logger.error(e)
# Update shard download state
self.__logger.error('First try failed. Retrying... (%s)' %
str(farmer_tries))
except requests.exceptions.Timeout as ret:
self.__logger.error('Request number %s for shard %s timed out.\
Took too much.' % (farmer_tries, shard_index))
self.__logger.error(ret)
except Exception as e:
self.__logger.error(e)
self.__logger.error('Unhandled error')
self.__logger.error('Error occured while downloading shard at '
'index %s. Retrying... (%s)' %
(shard_index, farmer_tries))
self.__logger.error('Shard download at index %s failed' % shard_index)
raise ClientError()
def getAsFile(self, chunk_size=512):
"""
Return the attachment as a TempFile
:param chunk_size:
:return: SpooledTemporaryFile
"""
tf = SpooledTemporaryFile(max_size=1024 * 1024, mode='w+b')
r = self._client.attachment._get_file(self.sys_id)
for chunk in r.iter_content(chunk_size=chunk_size):
tf.write(chunk)
tf.seek(0)
return tf
class TempInput(object):
def __init__(self, inputstr):
self.inputstr = inputstr
def __enter__(self):
self.tempfile = SpooledTemporaryFile()
self.tempfile.write(self.inputstr)
self.tempfile.seek(0)
return self.tempfile
def __exit__(self, type_, value, traceback):
self.tempfile.close()
return False
def buffer_iter(status_headers, iterator, buff_size=BUFF_SIZE * 4):
out = SpooledTemporaryFile(buff_size)
size = 0
for buff in iterator:
size += len(buff)
out.write(buff)
content_length_str = str(size)
# remove existing content length
status_headers.replace_header('Content-Length', content_length_str)
out.seek(0)
return StreamIter(out)
class TempInput(object):
def __init__(self, inputstr):
self.inputstr = inputstr
def __enter__(self):
self.tempfile = SpooledTemporaryFile()
self.tempfile.write(self.inputstr)
self.tempfile.seek(0)
return self.tempfile
def __exit__(self, type_, value, traceback):
self.tempfile.close()
return False
def _buffer_response(status_headers, iterator):
out = SpooledTemporaryFile(ProxyRouter.BUFF_RESPONSE_MEM_SIZE)
size = 0
for buff in iterator:
size += len(buff)
out.write(buff)
content_length_str = str(size)
# remove existing content length
status_headers.replace_header('Content-Length', content_length_str)
out.seek(0)
return RewriteContent.stream_to_gen(out)
def _buffer_response(status_headers, iterator):
out = SpooledTemporaryFile(ProxyRouter.BUFF_RESPONSE_MEM_SIZE)
size = 0
for buff in iterator:
size += len(buff)
out.write(buff)
content_length_str = str(size)
# remove existing content length
status_headers.replace_header('Content-Length',
content_length_str)
out.seek(0)
return RewriteContent.stream_to_gen(out)
def file_from_content(content):
"""Provides a real file object from file content
Converts ``FileStorage``, ``FileIntent`` and
``bytes`` to an actual file.
"""
f = content
if isinstance(content, cgi.FieldStorage):
f = content.file
elif isinstance(content, FileIntent):
f = content._fileobj
elif isinstance(content, byte_string):
f = SpooledTemporaryFile(INMEMORY_FILESIZE)
f.write(content)
f.seek(0)
return f
def file_from_content(content):
"""Provides a real file object from file content
Converts ``FileStorage``, ``FileIntent`` and
``bytes`` to an actual file.
"""
f = content
if isinstance(content, cgi.FieldStorage):
f = content.file
elif isinstance(content, FileIntent):
f = content._fileobj
elif isinstance(content, byte_string):
f = SpooledTemporaryFile(INMEMORY_FILESIZE)
f.write(content)
f.seek(0)
return f
def _save_content(self, obj, content, parameters):
# Seek our content back to the start
content.seek(0, os.SEEK_SET)
# Create a temporary file that will write to disk after a specified size
content_autoclose = SpooledTemporaryFile()
# Write our original content into our copy that will be closed by boto3
content_autoclose.write(content.read())
# Upload the object which will auto close the content_autoclose instance
super(S3DefaultStorage, self)._save_content(obj, content_autoclose, parameters)
# Cleanup if this is fixed upstream our duplicate should always close
if not content_autoclose.closed:
content_autoclose.close()
def request(self, url, params={}, headers={}):
"""
Retrieve SDMX messages.
If needed, override in subclasses to support other data providers.
:param url: The URL of the message.
:type url: str
:return: the xml data as file-like object
"""
# Generate current config. Merge in any given headers
cur_config = self.config.copy()
if 'headers' in cur_config:
cur_config['headers'] = cur_config['headers'].copy()
cur_config['headers'].update(headers)
else:
cur_config['headers'] = headers
with closing(requests.get(url, params=params,
**cur_config)) as response:
if response.status_code == requests.codes.OK:
# Prepare the temp file. xml content will be
# stored in a binary file, json in a textfile.
if (response.headers.get('Content-Type')
and ('json' in response.headers['Content-Type'])):
enc, fmode = response.encoding, 'w+t'
else:
enc, fmode = None, 'w+b'
# Create temp file ensuring 2to3 compatibility
if str_type == str: # we are on py3
source = STF(max_size=self.max_size,
mode=fmode,
encoding=enc)
else:
# On py27 we must omit the 'encoding' kwarg
source = STF(max_size=self.max_size, mode=fmode)
for c in response.iter_content(chunk_size=1000000,
decode_unicode=bool(enc)):
source.write(c)
else:
source = None
code = int(response.status_code)
if 400 <= code <= 499:
raise response.raise_for_status()
return source, response.url, response.headers, code
def gen_data(**kwargs):
"""Fetches realtime data and generates records"""
url = kwargs['BASE_URL']
# find the newest year
r = requests.get(url)
soup = BeautifulSoup(r.text, 'html.parser')
links = [link.get('href') for link in soup.find_all('a')]
year = sorted(filter(lambda l: l.startswith('2'), links))[-1]
url += year
# find the newest month
r = requests.get(url)
soup = BeautifulSoup(r.text, 'html.parser')
links = [link.get('href') for link in soup.find_all('a')]
month = sorted(filter(lambda l: l[0] in {'0', '1'}, links))[-1]
url += month
# find the newest file
r = requests.get(url)
soup = BeautifulSoup(r.text, 'html.parser')
links = [link.get('href') for link in soup.find_all('a')]
name = 'acled-all-africa-file'
filterer = lambda l: l.lower().startswith(name) and 'xls' in l
files = sorted(filter(filterer, links))
if files:
ftups = [(f.split('.')[0].split('-')[5], f) for f in files]
filename = sorted(ftups)[-1][1]
url += filename
# download the file
r = requests.get(url)
f = SpooledTemporaryFile() # wrap to access `fileno`
f.write(r.content)
records = io.read_xls(f, sanitize=True, encoding=r.encoding)
year = dt.now().year
filtered = it.ifilter(lambda r: int(float(r['year'])) == year, records)
for record in filtered:
month = parse(record['event_date']).month
month = '0%s' % month if month < 10 else month
record['year_month'] = '%s%s' % (year, month)
yield record
class OrcCompressor(object):
def __init__(self):
self._buffer = SpooledTemporaryFile(mode='wb')
def compress(self, data: bytes):
self._buffer.write(data)
def flush(self):
if self._buffer:
self._buffer.seek(0)
df = pd.read_csv(self._buffer)
with SpooledTemporaryFile(mode='wb') as wfd:
df.to_parquet(wfd, engine='pyarrow')
wfd.seek(0)
return wfd.read()
def __del__(self):
self._buffer.close()
def _send_boto3(self, boto3, resp, request):
from botocore.exceptions import BotoCoreError, ClientError
bucket_name, key_string = url_to_s3_info(request.url)
# Get the key without validation that it exists and that we have
# permissions to list its contents.
key = boto3.resource('s3').Object(bucket_name, key_string[1:])
try:
response = key.get()
except (BotoCoreError, ClientError) as exc:
# This exception will occur if the bucket does not exist or if the
# user does not have permission to list its contents.
resp.status_code = 404
message = {
"error": "error downloading file from s3",
"path": request.url,
"exception": repr(exc),
}
fh = SpooledTemporaryFile()
fh.write(ensure_binary(json.dumps(message)))
fh.seek(0)
resp.raw = fh
resp.close = resp.raw.close
return resp
key_headers = response['ResponseMetadata']['HTTPHeaders']
resp.headers = CaseInsensitiveDict({
"Content-Type":
key_headers.get('content-type', "text/plain"),
"Content-Length":
key_headers['content-length'],
"Last-Modified":
key_headers['last-modified'],
})
f = SpooledTemporaryFile()
key.download_fileobj(f)
f.seek(0)
resp.raw = f
resp.close = resp.raw.close
return resp
def buffer_iter(cls, orig_iter, buff_size=65536):
out = SpooledTemporaryFile(buff_size)
size = 0
for buff in orig_iter:
size += len(buff)
out.write(buff)
content_length_str = str(size)
out.seek(0)
def read_iter():
while True:
buff = out.read(buff_size)
if not buff:
break
yield buff
return content_length_str, read_iter()
def request(self, url, params={}, headers={}):
"""
Retrieve SDMX messages.
If needed, override in subclasses to support other data providers.
:param url: The URL of the message.
:type url: str
:return: the xml data as file-like object
"""
# Generate current config. Merge in any given headers
cur_config = self.config.copy()
if 'headers' in cur_config:
cur_config['headers'] = cur_config['headers'].copy()
cur_config['headers'].update(headers)
else:
cur_config['headers'] = headers
with closing(requests.get(url, params=params, **cur_config)) as response:
if response.status_code == requests.codes.OK:
# Prepare the temp file. xml content will be
# stored in a binary file, json in a textfile.
if (response.headers.get('Content-Type')
and ('json' in response.headers['Content-Type'])):
enc, fmode = response.encoding, 'w+t'
else:
enc, fmode = None, 'w+b'
# Create temp file ensuring 2to3 compatibility
if str_type == str: # we are on py3
source = STF(
max_size=self.max_size, mode=fmode, encoding=enc)
else:
# On py27 we must omit the 'encoding' kwarg
source = STF(max_size=self.max_size, mode=fmode)
for c in response.iter_content(chunk_size=1000000,
decode_unicode=bool(enc)):
source.write(c)
else:
source = None
code = int(response.status_code)
if 400 <= code <= 499:
raise response.raise_for_status()
return source, response.url, response.headers, code
def parseSMESH(self, filename):
vert_fn_ = filename + '.vertices'
face_fn_ = filename + '.faces'
verts = TemporaryFile(mode='w+r')
faces = TemporaryFile(mode='w+r')
fd = open( filename, 'r' )
fd = _skipCommentsAndEmptyLines( fd )
#skip one more line..
dummy = fd.readline()
first_vert = True
while fd:
line = fd.readline()
if line.startswith( '#' ):
continue
if len(line.split()) < self.dim + 0:
break
if first_vert:
self.zero_based_idx = line.startswith('0')
first_vert = False
verts.write(line)
print 'vertice writing complete'
fd = _skipCommentsAndEmptyLines( fd )
#skip one more line..
dummy = fd.readline()
while fd:
line = fd.readline()
if line.startswith( '#' ):
continue
if len(line.split()) < self.dim + 1:
break
faces.write(line)
print 'face writing complete'
#we need the cursor at top for individual parsing
verts.seek(0)
faces.seek(0)
_parseSMESH_vertices(self,verts)
print 'vert parsing complete'
_parseSMESH_faces(self, faces)
print 'face parsing complete'
self.buildAdjacencyList()
def _save_content(self, obj, content, parameters):
"""
We create a clone of the content file as when this is passed to boto3 it wrongly closes
the file upon upload where as the storage backend expects it to still be open
"""
# Seek our content back to the start
content.seek(0, os.SEEK_SET)
# Create a temporary file that will write to disk after a specified size
content_autoclose = SpooledTemporaryFile()
# Write our original content into our copy that will be closed by boto3
content_autoclose.write(content.read())
# Upload the object which will auto close the content_autoclose instance
super(CustomS3Boto3Storage, self)._save_content(obj, content_autoclose, parameters)
# Cleanup if this is fixed upstream our duplicate should always close
if not content_autoclose.closed:
content_autoclose.close()
class TempInput(object):
""" Acts as STDIN for a Popen process.
Initialize with the string you want to send on startup.
Enoding can be set with the optional 'encoding' arg.
"""
def __init__(self, inputstr, encoding=None):
self.encoding = encoding or 'utf-8'
self.inputstr = inputstr.encode(self.encoding)
def __enter__(self):
self.tempfile = SpooledTemporaryFile()
self.tempfile.write(self.inputstr)
self.tempfile.seek(0)
return self.tempfile
def __exit__(self, type_, value, traceback):
self.tempfile.close()
return False
def send_request(self, ip, port, cert, options):
s = socket(AF_INET, SOCK_STREAM)
s.setblocking(0)
s.settimeout(int(self.conf['Webserver']['web']['pcap_timeout']))
s_ssl = ssl.wrap_socket(s, ca_certs=cert.name, cert_reqs=ssl.CERT_REQUIRED, ssl_version=ssl.PROTOCOL_SSLv3)
try:
s_ssl.connect((ip, int(port)))
except error:
return 'Cannot connect to Receiver'
s_ssl.send(options)
s_ssl.send('END_EVENT')
tmp_file = SpooledTemporaryFile(mode='wb')
while True:
try:
data = s_ssl.recv(8192)
except ssl.SSLError:
return 'Request Timed Out'
if data == b'END_EVENT':
break
elif data == 'No Packets Found':
return 'No Packets Found'
elif data == 'Sensor cannot be reached':
return 'Sensor cannot be reached'
elif data == 'Request Timed Out':
return 'Request Timed Out'
else:
tmp_file.write(data)
tmp_file.seek(0)
cert.close()
s_ssl.close()
return tmp_file
def get_tile(self, tile, mapname=""):
if not self.has_tile(tile,mapname):
# Generate our empty tile
fileobj = tile.generate()
# Set it in cache
cachekey = self.key_for_tile(tile,mapname)
cache.set(cachekey, fileobj.read(), self.cache_time)
# Rewind our in-memory image pseudofile and return it
fileobj.seek(0)
return fileobj
else:
# Get data from cache
tile_data = self.get_tile_bytes(tile, mapname)
# Return it as a file-like object
tmpfile = SpooledTemporaryFile()
tmpfile.write(tile_data)
tmpfile.seek(0)
return tmpfile
def get_emptytile(self, tile):
tile_key = os.path.join("gheat", "empty_tiles", tile.color_scheme, "%s-%s-%s" % (tile.zoom, tile.x, tile.y))
if not cache.has_key(tile_key):
# Generate our empty tile
fileobj = tile.get_empty()
# Set it in cache
cache.set(tile_key, fileobj.read(), self.cache_time)
# Rewind our in-memory image pseudofile and return it
fileobj.seek(0)
return fileobj
else:
# Get data from cache
tile_data = self.get_emptytile_bytes(tile, mapname)
# Return it as a file-like object
tmpfile = SpooledTemporaryFile()
tmpfile.write(tile_data)
tmpfile.seek(0)
return tmpfile
def upload_to_s3(path, file, bucket_name=None):
s3 = get_boto_client()
bucket_name = bucket_name or current_app.config["AWS_S3_BUCKET"]
# Need to clone the file because boto3 autocloses the file after uploading to s3,
# preventing further use of the uploaded file.
file_s3 = SpooledTemporaryFile()
file_s3.write(file.read())
file_s3.seek(0)
s3.upload_fileobj(
file_s3,
bucket_name,
path,
ExtraArgs={"ACL": "public-read", "ContentType": file.content_type},
)
if not file_s3.closed:
file_s3.close()
return bucket_name, path
def _send_boto3(self, boto3, resp, request):
from botocore.exceptions import BotoCoreError, ClientError
bucket_name, key_string = url_to_s3_info(request.url)
# Get the key without validation that it exists and that we have
# permissions to list its contents.
key = boto3.resource('s3').Object(bucket_name, key_string[1:])
try:
response = key.get()
except (BotoCoreError, ClientError) as exc:
# This exception will occur if the bucket does not exist or if the
# user does not have permission to list its contents.
resp.status_code = 404
message = {
"error": "error downloading file from s3",
"path": request.url,
"exception": repr(exc),
}
fh = SpooledTemporaryFile()
fh.write(ensure_binary(json.dumps(message)))
fh.seek(0)
resp.raw = fh
resp.close = resp.raw.close
return resp
key_headers = response['ResponseMetadata']['HTTPHeaders']
resp.headers = CaseInsensitiveDict({
"Content-Type": key_headers.get('content-type', "text/plain"),
"Content-Length": key_headers['content-length'],
"Last-Modified": key_headers['last-modified'],
})
f = SpooledTemporaryFile()
key.download_fileobj(f)
f.seek(0)
resp.raw = f
resp.close = resp.raw.close
return resp
def graph(self):
if self.code is '':
return None
stdin = SpooledTemporaryFile()
stdout = SpooledTemporaryFile()
stdin.write('@startuml\n')
stdin.write(self.code)
stdin.write('@enduml\n')
stdin.seek(0)
args = [
self.java,
'-jar',
self.jar,
'-p',
'-tdot',
]
p = Popen(args, stdin=stdin, stdout=stdout)
if p.wait() != 0:
return None
stdout.seek(0)
graph = stdout.read()
return graph_from_dot_data(graph)
def _create_dot_file(self, edge_pairs):
'''Create a file suitable for use with dot.
@param edge_pairs: A list of tuples of edges to be rendered.
@return: A file object containing contents to be rendered by dot.
'''
file = SpooledTemporaryFile(mode = 'w')
file.write('digraph G {\n')
unique_nodes = Set()
def add_node(node):
if node not in unique_nodes:
attributes = '[id={id},label="{label}"]' \
.format(id = node.get_identifier(),
label = repr(node))
file.write('{id} {attrs}\n' \
.format(id = node.get_identifier(),
attrs = attributes))
for src, dst in edge_pairs:
if dst is None:
add_node(src)
continue
add_node(src)
add_node(dst)
unique_nodes.add(src)
unique_nodes.add(dst)
file.write('{src} -> {dst} {attr}\n' \
.format(src = src.get_identifier(),
dst = dst.get_identifier(),
attr = ''))
file.write('}\n')
file.seek(0)
return file
def process(self, host, s):
encrypted_options = ''
while True:
data = s.recv(8192)
if data == b'END_EVENT':
break
else:
encrypted_options = encrypted_options + data
try:
options = self.decrypt_options(encrypted_options)
except:
s.close()
return
soc = socket(AF_INET, SOCK_STREAM)
soc.setblocking(0)
soc.settimeout(int(self.config['Event_Receiver']['PCAP']['timeout']))
db = self.core.get_db()
sensors = db.sensors
client_info = sensors.find_one( { "SERVER": options['sensor'] })
client_cert = client_info['cert']
cert_tmp = NamedTemporaryFile(mode='w+b', suffix='.pem')
cert_tmp.write(client_cert)
cert_tmp.flush()
soc_ssl = ssl.wrap_socket(soc, ca_certs=cert_tmp.name, cert_reqs=ssl.CERT_REQUIRED, ssl_version=ssl.PROTOCOL_SSLv3)
encrypted_options = self.encrypt_requests(self.config, options)
try:
soc_ssl.connect((client_info['IP'], int(client_info['sensor_port'])))
except error:
s.send('Sensor cannot be reached')
s.close()
return
soc_ssl.send(encrypted_options)
soc_ssl.send('END_EVENT')
tmp_file = SpooledTemporaryFile(mode='wb')
while True:
try:
data = soc_ssl.recv(8192)
except ssl.SSLError:
s.send('Request Timed Out')
s.close()
return
if data == b'END_EVENT':
break
elif data != 'No Packets Found':
tmp_file.write(data)
else:
s.send('No Packets Found')
s.close()
cert_tmp.close()
return
tmp_file.seek(0)
pcap = tmp_file.read(8192)
try:
while (pcap):
s.send(pcap)
pcap = tmp_file.read(8192)
s.send(b'END_EVENT')
except error:
s.close()
return
s.close()
cert_tmp.close()
return
