async def hashset(self, sha256):
'''
Calculate additional hashes for a file in the Axon.
Args:
sha256 (bytes): The sha256 hash of the file in bytes.
Returns:
dict: A dictionary containing hashes of the file.
'''
if not await self.has(sha256):
raise s_exc.NoSuchFile(
mesg='Axon does not contain the requested file.',
sha256=s_common.ehex(sha256))
fhash = s_common.ehex(sha256)
logger.debug(f'Getting blob [{fhash}].',
extra=await self.getLogExtra(sha256=fhash))
hashset = s_hashset.HashSet()
async for byts in self._get(sha256):
hashset.update(byts)
await asyncio.sleep(0)
return dict([(n, s_common.ehex(h)) for (n, h) in hashset.digests()])
async def test_axon_wput(self):
async with self.getTestCore() as core:
axon = core.axon
axon.addHttpApi('/api/v1/pushfile', HttpPushFile, {'cell': axon})
async with await axon.upload() as fd:
await fd.write(b'asdfasdf')
size, sha256 = await fd.save()
host, port = await axon.addHttpsPort(0, host='127.0.0.1')
async with axon.getLocalProxy() as proxy:
resp = await proxy.wput(sha256, f'https://127.0.0.1:{port}/api/v1/pushfile', method='PUT', ssl=False)
self.eq(True, resp['ok'])
self.eq(200, resp['code'])
opts = {'vars': {'sha256': s_common.ehex(sha256)}}
q = f'return($lib.axon.wput($sha256, "https://127.0.0.1:{port}/api/v1/pushfile", ssl=(0)))'
resp = await core.callStorm(q, opts=opts)
self.eq(True, resp['ok'])
self.eq(200, resp['code'])
opts = {'vars': {'sha256': s_common.ehex(s_common.buid())}}
resp = await core.callStorm(q, opts=opts)
self.eq(False, resp['ok'])
self.isin('Axon does not contain the requested file.', resp.get('mesg'))
q = f'''
$fields = $lib.list(
$lib.dict(name=file, sha256=$sha256, filename=file),
$lib.dict(name=zip_password, value=test)
)
$resp = $lib.inet.http.post("https://127.0.0.1:{port}/api/v1/pushfile",
fields=$fields, ssl_verify=(0))
return($resp)
'''
opts = {'vars': {'sha256': s_common.ehex(sha256)}}
resp = await core.callStorm(q, opts=opts)
self.true(resp['ok'])
self.eq(200, resp['code'])
opts = {'vars': {'sha256': s_common.ehex(s_common.buid())}}
resp = await core.callStorm(q, opts=opts)
self.false(resp['ok'])
self.isin('Axon does not contain the requested file.', resp.get('err'))
async with axon.getLocalProxy() as proxy:
resp = await proxy.postfiles(fields, f'https://127.0.0.1:{port}/api/v1/pushfile', ssl=False)
self.true(resp['ok'])
self.eq(200, resp['code'])
conf = {'http:proxy': 'socks5://user:[email protected]:1'}
async with self.getTestAxon(conf=conf) as axon:
async with axon.getLocalProxy() as proxy:
resp = await proxy.postfiles(fields, f'https://127.0.0.1:{port}/api/v1/pushfile', ssl=False)
self.false(resp['ok'])
self.isin('Can not connect to proxy 127.0.0.1:1', resp.get('err', ''))
async def get(self, sha256):
'''
Get bytes of a file.
Args:
sha256 (bytes): The sha256 hash of the file in bytes.
Examples:
Get the bytes from an Axon and process them::
buf = b''
async for bytz in axon.get(sha256):
buf =+ bytz
await dostuff(buf)
Yields:
bytes: Chunks of the file bytes.
Raises:
synapse.exc.NoSuchFile: If the file does not exist.
'''
if not await self.has(sha256):
raise s_exc.NoSuchFile(
mesg='Axon does not contain the requested file.',
sha256=s_common.ehex(sha256))
fhash = s_common.ehex(sha256)
logger.debug(f'Getting blob [{fhash}].',
extra=await self.getLogExtra(sha256=fhash))
async for byts in self._get(sha256):
yield byts
async def _handle_stat(self, core, opts):
''' Prints details about a particular cron job. Not actually a different API call '''
prefix = opts.prefix
crons = await core.listCronJobs()
idens = [cron[0] for cron in crons]
matches = [
iden for iden in idens if s_common.ehex(iden).startswith(prefix)
]
if len(matches) == 0:
self.printf(
'Error: provided iden does not match any valid authorized cron job'
)
return
elif len(matches) > 1:
self.printf('Error: provided iden matches more than one cron job')
return
iden = matches[0]
cron = [cron[1] for cron in crons if cron[0] == iden][0]
idenf = s_common.ehex(iden)
user = cron.get('username') or '<None>'
query = cron.get('query') or '<missing>'
isrecur = 'Yes' if cron.get('recur') else 'No'
startcount = cron.get('startcount') or 0
recs = cron.get('recs', [])
laststart = cron.get('laststarttime')
lastend = cron.get('lastfinishtime')
laststart = 'Never' if laststart is None else self._format_timestamp(
laststart)
lastend = 'Never' if lastend is None else self._format_timestamp(
lastend)
lastresult = cron.get('lastresult') or '<None>'
self.printf(f'iden: {idenf}')
self.printf(f'user: {user}')
self.printf(f'recurring: {isrecur}')
self.printf(f'# starts: {startcount}')
self.printf(f'last start time: {laststart}')
self.printf(f'last end time: {lastend}')
self.printf(f'last result: {lastresult}')
self.printf(f'query: {query}')
if not recs:
self.printf(f'entries: <None>')
else:
self.printf(
f'entries: {"incunit":10} {"incval":6} {"required"}')
for reqdict, incunit, incval in recs:
reqdict = reqdict or '<None>'
incunit = incunit or '<None>'
incval = incval or '<None>'
self.printf(
f' {incunit:10} {incval:6} {reqdict}')
async def runCmdOpts(self, opts):
if not os.path.isfile(opts.filepath):
self.printf(f'no such file: {opts.filepath}')
return
self.printf(f'uploading file: {opts.filepath}')
async with await self._cmd_cli.item.getAxonUpload() as upload:
with open(opts.filepath, 'rb') as fd:
byts = fd.read(10000000)
while byts:
await upload.write(byts)
byts = fd.read(10000000)
size, sha256 = await upload.save()
opts = {
'vars': {
'sha256': s_common.ehex(sha256),
'name': os.path.basename(opts.filepath),
}
}
await self._cmd_cli.storm(
'[ file:bytes=$sha256 ] { -:name [:name=$name] }', opts=opts)
async def _populate(self, sha256, genr, size=None):
'''
Populates the metadata and save the data itself if genr is not None
'''
assert genr is not None or size is not None
self._reqBelowLimit()
async with self.holdHashLock(sha256):
byts = self.axonslab.get(sha256, db=self.sizes)
if byts is not None:
return int.from_bytes(byts, 'big')
fhash = s_common.ehex(sha256)
logger.debug(f'Saving blob [{fhash}].',
extra=await self.getLogExtra(sha256=fhash))
if genr is not None:
size = await self._saveFileGenr(sha256, genr)
self._addSyncItem((sha256, size))
await self.axonmetrics.set('file:count',
self.axonmetrics.get('file:count') + 1)
await self.axonmetrics.set(
'size:bytes',
self.axonmetrics.get('size:bytes') + size)
self.axonslab.put(sha256, size.to_bytes(8, 'big'), db=self.sizes)
return size
async def _handle_list(self, core, opts):
triglist = await core.listTriggers()
if not triglist:
self.printf('No triggers found')
return
self.printf(
f'{"user":10} {"iden":12} {"cond":9} {"object":14} {"":10} {"storm query"}'
)
for iden, trig in triglist:
idenf = s_common.ehex(iden)[:8] + '..'
user = trig.get('user') or '<None>'
query = trig.get('storm') or '<missing>'
cond = trig.get('cond') or '<missing'
if cond.startswith('tag:'):
tag = '#' + (trig.get('tag') or '<missing>')
form = trig.get('form') or ''
obj, obj2 = form, tag
else:
obj, obj2 = trig.get('prop') or trig.get(
'form') or '<missing>', ''
self.printf(
f'{user:10} {idenf:12} {cond:9} {obj:14} {obj2:10} {query}')
async def test_storm_node_iden(self):
async with self.getTestCore() as core:
nodes = await core.nodes(
'[ test:int=10 test:str=$node.iden() ] +test:str')
iden = s_common.ehex(s_common.buid(('test:int', 10)))
self.eq(nodes[0].ndef, ('test:str', iden))
self.len(1, nodes)
async def test_it_reveng(self):
async with self.getTestCore() as core:
baseFile = s_common.ehex(s_common.buid())
fva = 0x404438
fopt = {'vars': {'file': baseFile,
'func': s_common.guid(),
'fva': fva}}
vstr = 'VertexBrandArtisanalBinaries'
sopt = {'vars': {'func': fopt['vars']['func'],
'string': vstr}}
fnode = await core.eval('[it:reveng:filefunc=($file, $func) :va=$fva]', opts=fopt).list()
snode = await core.eval('[it:reveng:funcstr=($func, $string)]', opts=sopt).list()
self.len(1, fnode)
self.eq(f'sha256:{baseFile}', fnode[0].get('file'))
self.eq(fva, fnode[0].get('va'))
self.len(1, snode)
self.eq(fnode[0].get('function'), snode[0].get('function'))
self.eq(vstr, snode[0].get('string'))
funcnode = await core.eval('it:reveng:function [ :name="FunkyFunction" :description="Test Function" ]').list()
self.len(1, funcnode)
self.eq("FunkyFunction", funcnode[0].get('name'))
self.eq("Test Function", funcnode[0].get('description'))
nodes = await core.eval(f'file:bytes={baseFile} -> it:reveng:filefunc :function -> it:reveng:funcstr:function').list()
self.len(1, nodes)
self.eq(vstr, nodes[0].get('string'))
async def del_(self, sha256):
'''
Remove the given bytes from the Axon by sha256.
Args:
sha256 (bytes): The sha256, in bytes, to remove from the Axon.
Returns:
boolean: True if the file is removed; false if the file is not present.
'''
async with self.holdHashLock(sha256):
byts = self.axonslab.pop(sha256, db=self.sizes)
if not byts:
return False
fhash = s_common.ehex(sha256)
logger.debug(f'Deleting blob [{fhash}].',
extra=await self.getLogExtra(sha256=fhash))
size = int.from_bytes(byts, 'big')
await self.axonmetrics.set('file:count',
self.axonmetrics.get('file:count') - 1)
await self.axonmetrics.set(
'size:bytes',
self.axonmetrics.get('size:bytes') - size)
await self._delBlobByts(sha256)
return True
async def _scheduleLoop(self):
'''
Task loop to issue query tasks at the right times.
'''
while True:
try:
timeout = None if not self.apptheap else self.apptheap[
0].nexttime - time.time()
if timeout is None or timeout >= 0.0:
await asyncio.wait_for(self._wake_event.wait(),
timeout=timeout)
except asyncio.TimeoutError:
pass
if self.isfini:
return
self._wake_event.clear()
now = time.time()
while self.apptheap and self.apptheap[0].nexttime <= now:
appt = heapq.heappop(self.apptheap)
appt.updateNexttime(now)
if appt.nexttime:
heapq.heappush(self.apptheap, appt)
if not appt.enabled:
continue
if appt.isrunning:
logger.warning(
'Appointment %s is still running from previous time when scheduled to run. Skipping.',
s_common.ehex(appt.iden))
else:
await self._execute(appt)
async def _handle_list(self, core, opts):
cronlist = await core.listCronJobs()
if not cronlist:
self.printf('No cron jobs found')
return
self.printf(
f'{"user":10} {"iden":10} {"recurs?":7} {"now?":4} '
f'{"# start":7} {"last start":16} {"last end":16} {"query"}')
for iden, cron in cronlist:
idenf = s_common.ehex(iden)[:8] + '..'
user = cron.get('username') or '<None>'
query = cron.get('query') or '<missing>'
isrecur = 'Y' if cron.get('recur') else 'N'
isrunning = 'Y' if cron.get('isrunning') else 'N'
startcount = cron.get('startcount') or 0
laststart = cron.get('laststarttime')
laststart = 'Never' if laststart is None else self._format_timestamp(
laststart)
lastend = cron.get('lastfinishtime')
lastend = 'Never' if lastend is None else self._format_timestamp(
lastend)
self.printf(
f'{user:10} {idenf:10} {isrecur:7} {isrunning:4} {startcount:7} {laststart:16} {lastend:16} {query}'
)
async def _load_all(self):
'''
Load all the appointments from persistent storage
'''
to_delete = []
for idenf, val in self._hivedict.items():
try:
iden = s_common.uhex(idenf)
appt = _Appt.unpack(val)
if appt.iden != iden:
raise s_exc.InconsistentStorage(mesg='iden inconsistency')
self._addappt(iden, appt)
self._next_indx = max(self._next_indx, appt.indx + 1)
except (s_exc.InconsistentStorage, s_exc.BadTime, TypeError,
KeyError) as e:
logger.warning(
'Invalid appointment %r found in storage: %r. Removing',
iden, e)
to_delete.append(iden)
continue
for iden in to_delete:
await self._hivedict.pop(s_common.ehex(iden))
# Make sure we don't assign the same index to 2 appointments
if self.appts:
maxindx = max(appt.indx for appt in self.appts.values())
self._next_indx = maxindx + 1
def migrate_v0_rules(self):
'''
Remove any v0 (i.e. pre-010) rules from storage and replace them with v1 rules.
Notes:
v0 had two differences user was a username. Replaced with iden of user as 'iden' field.
Also 'iden' was storage as binary. Now it is stored as hex string.
'''
for iden, valu in self.core.slab.scanByFull(db=self.trigdb):
ruledict = s_msgpack.un(valu)
ver = ruledict.get('ver')
if ver != 0:
continue
user = ruledict.pop('user')
if user is None:
logger.warning('Username missing in stored trigger rule %r',
iden)
continue
# In v0, stored user was username, in >0 user is useriden
user = self.core.auth.getUserByName(user).iden
if user is None:
logger.warning(
'Unrecognized username in stored trigger rule %r', iden)
continue
ruledict['ver'] = 1
ruledict['useriden'] = user
newiden = s_common.ehex(iden)
self.core.slab.pop(iden, db=self.trigdb)
self.core.slab.put(newiden.encode(),
s_msgpack.en(ruledict),
db=self.trigdb)
async def _runJob(self, user, appt):
'''
Actually run the storm query, updating the appropriate statistics and results
'''
count = 0
appt.isrunning = True
appt.laststarttime = time.time()
appt.startcount += 1
await self._storeAppt(appt)
idenf = s_common.ehex(appt.iden)
logger.info(
f'Agenda executing for iden={idenf}, user={user}: query={{appt.query}}'
)
try:
async for _ in self.core.eval(appt.query, user=user): # NOQA
count += 1
except asyncio.CancelledError:
result = 'cancelled'
raise
except Exception as e:
result = f'raised exception {e}'
logger.exception('Agenda job %s raised exception', idenf)
else:
result = f'finished successfully with {count} nodes'
finally:
finishtime = time.time()
logger.info(
f'Agenda completed query for iden={idenf} with result="{result}" took {finishtime:0.2}'
)
appt.lastfinishtime = finishtime
appt.isrunning = False
appt.lastresult = result
if not self.isfini:
await self._storeAppt(appt)
def migrate_v0_rules(self):
'''
Remove any v0 (i.e. pre-010) rules from storage and replace them with v1 rules.
Notes:
v0 had two differences user was a username. Replaced with iden of user as 'iden' field.
Also 'iden' was storage as binary. Now it is stored as hex string.
'''
for iden, valu in self.core.slab.scanByFull(db=self.trigdb):
ruledict = s_msgpack.un(valu)
ver = ruledict.get('ver')
if ver != 0:
continue
user = ruledict.pop('user')
if user is None:
logger.warning('Username missing in stored trigger rule %r', iden)
continue
# In v0, stored user was username, in >0 user is useriden
user = self.core.auth.getUserByName(user).iden
if user is None:
logger.warning('Unrecognized username in stored trigger rule %r', iden)
continue
ruledict['ver'] = 1
ruledict['useriden'] = user
newiden = s_common.ehex(iden)
self.core.slab.pop(iden, db=self.trigdb)
self.core.slab.put(newiden.encode(), s_msgpack.en(ruledict), db=self.trigdb)
async def get(self, sha256):
if not await self.has(sha256):
raise s_exc.NoSuchFile(sha256=s_common.ehex(sha256))
for lkey, byts in self.blobslab.scanByPref(sha256, db=self.blobs):
yield byts
async def get(self, sha256):
if not await self.has(sha256):
raise s_exc.NoSuchFile(sha256=s_common.ehex(sha256))
for lkey, byts in self.blobslab.scanByPref(sha256, db=self.blobs):
yield byts
def stor(self):
'''
Writes the current provenance stack to storage if it wasn't already there
Returns (iden, provstack) if was written, (None, None) if it was already there
'''
if not ProvenanceEnabled:
return None, None
iden, waswritten, provstack = get()
if waswritten:
return None, None
assert iden is not None
# Convert each frame back from (k, v) tuples to a dict
misc, frames = provstack
dictframes = [(typ, {k: v
for (k, v) in info}) for (typ, info) in frames]
bytz = s_msgpack.en((misc, dictframes))
didwrite = self.slab.put(iden, bytz, overwrite=False, db=self.db)
if didwrite:
self.provseq.save([iden])
setiden(iden, True)
return s_common.ehex(iden), provstack
def test_common_ehex_uhex(self):
byts = b'deadb33f00010203'
s = s_common.ehex(byts)
self.isinstance(s, str)
self.eq(s, '64656164623333663030303130323033')
# uhex is a linear transform back
obyts = s_common.uhex(s)
self.isinstance(obyts, bytes)
self.eq(byts, obyts)
def test_common_ehex_uhex(self):
byts = b'deadb33f00010203'
s = s_common.ehex(byts)
self.isinstance(s, str)
self.eq(s, '64656164623333663030303130323033')
# uhex is a linear transform back
obyts = s_common.uhex(s)
self.isinstance(obyts, bytes)
self.eq(byts, obyts)
async def get(self, sha256):
if not await self.has(sha256):
raise s_exc.NoSuchFile(
mesg='Axon does not contain the requested file.',
sha256=s_common.ehex(sha256))
async for byts in self._get(sha256):
yield byts
def commit(self):
'''
Writes the current provenance stack to storage if it wasn't already there and returns it
Returns (Tuple[bool, str, List[]]):
Whether the stack was not cached, the iden of the prov stack, and the provstack
'''
providen, provstack = get()
wasnew = (providen is None)
if wasnew:
providen = self.getProvIden(provstack)
setiden(providen)
return wasnew, s_common.ehex(providen), provstack
def commit(self):
'''
Writes the current provenance stack to storage if it wasn't already there and returns it
Returns (Tuple[bool, str, List[]]):
Whether the stack was not cached, the iden of the prov stack, and the provstack
'''
providen, provstack = get()
wasnew = (providen is None)
if wasnew:
providen = self.getProvIden(provstack)
setiden(providen)
return wasnew, s_common.ehex(providen), provstack
async def test_modification_persistence(self):
with self.getTestDir() as fdir:
async with self.getTestCore(dirn=fdir) as core:
rootiden = core.auth.getUserByName('root').iden
core.view.triggers.add('root',
'node:add',
'[inet:user=1] | testcmd',
info={'form': 'inet:ipv4'})
triggers = core.view.triggers.list()
self.eq(triggers[0][1].storm, '[inet:user=1] | testcmd')
iden = triggers[0][0]
core.view.triggers.mod(iden,
'[inet:user=2 .test:univ=4] | testcmd')
triggers = core.view.triggers.list()
self.eq(triggers[0][1].storm,
'[inet:user=2 .test:univ=4] | testcmd')
# Sad case
self.raises(s_exc.BadSyntax, core.view.triggers.mod, iden,
' | | badstorm ')
self.raises(s_exc.NoSuchIden, core.view.triggers.mod,
'deadb33f', 'inet:user')
# Manually store a v0 trigger
ruledict = {
'ver': 0,
'cond': 'node:add',
'form': 'inet:ipv4',
'user': '******',
'storm': 'testcmd'
}
v0iden = b'\xff' * 16
core.slab.put(v0iden,
s_msgpack.en(ruledict),
db=core.trigstor.trigdb)
async with self.getTestCore(dirn=fdir) as core:
triggers = core.view.triggers.list()
self.len(2, triggers)
self.eq(triggers[0][1].storm,
'[inet:user=2 .test:univ=4] | testcmd')
# Verify that the v0 trigger was migrated correctly
iden2, trig2 = triggers[1]
self.eq(iden2, s_common.ehex(v0iden))
self.eq(trig2.ver, 1)
self.eq(trig2.storm, 'testcmd')
self.eq(trig2.useriden, rootiden)
async def getEmbeds(self, embeds):
'''
Return a dictionary of property embeddings.
'''
retn = {}
cache = {}
async def walk(n, p):
valu = n.props.get(p)
if valu is None:
return None
prop = n.form.prop(p)
if prop is None:
return None
if prop.modl.form(prop.type.name) is None:
return None
buid = s_common.buid((prop.type.name, valu))
step = cache.get(buid, s_common.novalu)
if step is s_common.novalu:
step = cache[buid] = await node.snap.getNodeByBuid(buid)
return step
for nodepath, relprops in embeds.items():
steps = nodepath.split('::')
node = self
for propname in steps:
node = await walk(node, propname)
if node is None:
break
if node is None:
continue
embdnode = retn.get(nodepath)
if embdnode is None:
embdnode = retn[nodepath] = {}
embdnode['*'] = s_common.ehex(node.buid)
for relp in relprops:
embdnode[relp] = node.props.get(relp)
return retn
def precommit(self):
'''
Determine the iden for the current provenance stack and return it
Returns the iden
'''
if not ProvenanceEnabled:
return None
providen, waswritten, provstack = get()
if providen is None:
providen = _providen(provstack)
setiden(providen, False)
return s_common.ehex(providen)
async def _libBytesPut(self, byts):
'''
Save the given bytes variable to the axon.
Returns:
($size, $sha256)
Example:
($size, $sha2) = $lib.bytes.put($bytes)
'''
if not isinstance(byts, bytes):
mesg = '$lib.bytes.put() requires a bytes argument'
raise s_exc.BadArg(mesg=mesg)
await self.runt.snap.core.axready.wait()
size, sha2 = await self.runt.snap.core.axon.put(byts)
return (size, s_common.ehex(sha2))
async def _match_idens(self, core, prefix):
'''
Returns the iden that starts with prefix. Prints out error and returns None if it doesn't match
exactly one.
'''
idens = [iden for iden, trig in await core.listCronJobs()]
matches = [
iden for iden in idens if s_common.ehex(iden).startswith(prefix)
]
if len(matches) == 1:
return matches[0]
elif len(matches) == 0:
self.printf(
'Error: provided iden does not match any valid authorized cron job'
)
else:
self.printf('Error: provided iden matches more than one cron job')
return None
async def test_storm_yieldvalu(self):
async with self.getTestCore() as core:
nodes = await core.nodes('[ inet:ipv4=1.2.3.4 ]')
buid0 = nodes[0].buid
iden0 = s_common.ehex(buid0)
nodes = await core.nodes('yield $foo', opts={'vars': {'foo': (iden0,)}})
self.len(1, nodes)
self.eq(nodes[0].ndef, ('inet:ipv4', 0x01020304))
def genr():
yield iden0
async def agenr():
yield iden0
nodes = await core.nodes('yield $foo', opts={'vars': {'foo': (iden0,)}})
self.len(1, nodes)
self.eq(nodes[0].ndef, ('inet:ipv4', 0x01020304))
nodes = await core.nodes('yield $foo', opts={'vars': {'foo': buid0}})
self.len(1, nodes)
self.eq(nodes[0].ndef, ('inet:ipv4', 0x01020304))
nodes = await core.nodes('yield $foo', opts={'vars': {'foo': genr()}})
self.len(1, nodes)
self.eq(nodes[0].ndef, ('inet:ipv4', 0x01020304))
nodes = await core.nodes('yield $foo', opts={'vars': {'foo': agenr()}})
self.len(1, nodes)
self.eq(nodes[0].ndef, ('inet:ipv4', 0x01020304))
nodes = await core.nodes('yield $foo', opts={'vars': {'foo': nodes[0]}})
self.len(1, nodes)
self.eq(nodes[0].ndef, ('inet:ipv4', 0x01020304))
nodes = await core.nodes('yield $foo', opts={'vars': {'foo': None}})
self.len(0, nodes)
with self.raises(s_exc.BadLiftValu):
await core.nodes('yield $foo', opts={'vars': {'foo': 'asdf'}})
async def test_axon_wget(self):
async with self.getTestAxon() as axon:
visi = await axon.auth.addUser('visi')
await visi.setAdmin(True)
await visi.setPasswd('secret')
async with await axon.upload() as fd:
await fd.write(b'asdfasdf')
size, sha256 = await fd.save()
host, port = await axon.addHttpsPort(0, host='127.0.0.1')
sha2 = s_common.ehex(sha256)
async with axon.getLocalProxy() as proxy:
resp = await proxy.wget(f'https://*****:*****@127.0.0.1:{port}/api/v1/axon/files/by/sha256/{sha2}',
ssl=False)
self.eq(True, resp['ok'])
self.eq(200, resp['code'])
self.eq(8, resp['size'])
self.eq('application/octet-stream', resp['headers']['Content-Type'])
resp = await proxy.wget(f'http://*****:*****@127.0.0.1:{port}/api/v1/axon/files/by/sha256/{sha2}')
self.false(resp['ok'])
async def timeout(self):
await asyncio.sleep(2)
with mock.patch.object(s_httpapi.ActiveV1, 'get', timeout):
resp = await proxy.wget(f'https://*****:*****@127.0.0.1:{port}/api/v1/active', timeout=1,
ssl=False)
self.eq(False, resp['ok'])
self.eq('TimeoutError', resp['mesg'])
conf = {'http:proxy': 'socks5://user:[email protected]:1'}
async with self.getTestAxon(conf=conf) as axon:
async with axon.getLocalProxy() as proxy:
resp = await proxy.wget('http://vertex.link')
self.isin('Can not connect to proxy 127.0.0.1:1', resp.get('mesg', ''))
async def delete(self, iden):
'''
Delete an appointment
'''
appt = self.appts.get(iden)
if appt is None:
raise s_exc.NoSuchIden()
try:
heappos = self.apptheap.index(appt)
except ValueError:
pass # this is OK, just a non-recurring appt that has no more records
else:
# If we're already the last item, just delete it
if heappos == len(self.apptheap) - 1:
del self.apptheap[heappos]
else:
# put the last item at the current position and reheap
self.apptheap[heappos] = self.apptheap.pop()
heapq.heapify(self.apptheap)
del self.appts[iden]
await self._hivedict.pop(s_common.ehex(iden))
async def save(self, sha256, genr):
'''
Save a generator of bytes to the Axon.
Args:
sha256 (bytes): The sha256 hash of the file in bytes.
genr: The bytes generator function.
Returns:
int: The size of the bytes saved.
'''
self._reqBelowLimit()
async with self.holdHashLock(sha256):
byts = self.axonslab.get(sha256, db=self.sizes)
if byts is not None:
return int.from_bytes(byts, 'big')
fhash = s_common.ehex(sha256)
logger.debug(f'Saving blob [{fhash}].',
extra=await self.getLogExtra(sha256=fhash))
size = await self._saveFileGenr(sha256, genr)
self._addSyncItem((sha256, size))
await self.axonmetrics.set('file:count',
self.axonmetrics.get('file:count') + 1)
await self.axonmetrics.set(
'size:bytes',
self.axonmetrics.get('size:bytes') + size)
self.axonslab.put(sha256, size.to_bytes(8, 'big'), db=self.sizes)
return size
async def main(argv, outp=None):
if outp is None: # pragma: no cover
outp = s_output.OutPut()
path = s_common.getSynPath('telepath.yaml')
telefini = await s_telepath.loadTeleEnv(path)
pars = makeargparser()
opts = pars.parse_args(argv)
axon = await s_telepath.openurl(opts.axon)
core = None
if opts.cortex:
core = await s_telepath.openurl(opts.cortex)
tags = set()
if opts.tags:
for tag in opts.tags.split(','):
tags.add(tag)
tags = tuple(tags)
if tags:
outp.printf(f'adding tags: {tags}')
filepaths = set()
for item in opts.filenames:
paths = glob.glob(item, recursive=opts.recursive)
if not paths:
outp.printf(f'filepath does not contain any files: {item}')
continue
filepaths.update([path for path in paths if os.path.isfile(path)])
for path in filepaths:
bname = os.path.basename(path)
hset = s_hashset.HashSet()
with s_common.reqfile(path) as fd:
hset.eatfd(fd)
fhashes = {htyp: hasher.hexdigest() for htyp, hasher in hset.hashes}
sha256 = fhashes.get('sha256')
bsha256 = s_common.uhex(sha256)
if not await axon.has(bsha256):
async with await axon.upload() as upfd:
with s_common.genfile(path) as fd:
for byts in s_common.iterfd(fd):
await upfd.write(byts)
size, hashval = await upfd.save()
if hashval != bsha256: # pragma: no cover
raise s_exc.SynErr(mesg='hashes do not match',
ehash=s_common.ehex(hashval),
ahash=hashval)
outp.printf(f'Uploaded [{bname}] to axon')
else:
outp.printf(f'Axon already had [{bname}]')
if core:
opts = {
'vars': {
'md5': fhashes.get('md5'),
'sha1': fhashes.get('sha1'),
'sha256': fhashes.get('sha256'),
'size': hset.size,
'name': bname,
'tags': tags,
}
}
q = '[file:bytes=$sha256 :md5=$md5 :sha1=$sha1 :size=$size :name=$name] ' \
'{ for $tag in $tags { [+#$tag] } }'
msgs = await core.storm(q, opts=opts).list()
node = [m[1] for m in msgs if m[0] == 'node'][0]
iden = node[0][1]
size = node[1]['props']['size']
name = node[1]['props']['name']
mesg = f'file: {bname} ({size}) added to core ({iden}) as {name}'
outp.printf(mesg)
await axon.fini()
if core:
await core.fini()
if telefini: # pragma: no cover
await telefini()
return 0
def iden(self):
return s_common.ehex(self.buid)
