def test_case_checkers():
test = "lower_snake_case"
assert utils.get_type(test) == "lower snake"
test = "UPPER_SNAKE_CASE"
assert utils.get_type(test) == "upper snake"
test = "lowerCamelCase"
assert utils.get_type(test) == "lower camel"
test = "UpperCamelCase"
assert utils.get_type(test) == "upper camel"
test = "any other"
assert utils.get_type(test) == "other"
def __init__(self):
utils.log("Started service")
revision = utils.get_revision()
utils.log("Board revision: {}".format(revision))
if revision is not None:
utils.set_property_setting('revision', revision)
max_ram = utils.get_max_ram()
utils.log("RAM size: {}MB".format(max_ram))
utils.set_property_setting('max_ram', max_ram)
board_type = utils.get_type()
utils.log("Board type: {}".format(board_type))
if board_type is not None:
utils.set_property_setting('type', board_type)
try:
utils.maybe_init_settings()
except IOError:
utils.log_exception()
self.monitor = MyMonitor(updated_settings_callback=self.apply_config)
while (not xbmc.abortRequested):
xbmc.sleep(1000)
async def send_message(self, update: dict):
"""
Generates and sends a Discord message
"""
android = update['android']
codename = update['codename']
device = update['device']
filename = update['filename']
filesize = update['size']
version = update['version']
download = update['download']
branch = get_branch(version)
region = get_region(filename, codename, version)
rom_type = get_type(filename)
codename = codename.split('_')[0]
device = device.replace(f' {region}', '')
desc = f"**Device**: {device} \n" \
f"**Codename**: `{codename}` \n" \
f"**Region**: {region} \n" \
f"**Version**: `{version} | {android}` \n" \
f"**Size**: {filesize} \n" \
f"**Download**: [Here]({download})"
embed = discord.Embed(title=f"New {branch} {rom_type} update available!",
color=discord.Colour.orange(), description=desc)
embed.set_footer(text=f"https://xiaomifirmwareupdater.com/miui/{codename}")
device = device.lower()
for name, channel in self.channels.items():
if device.startswith(name):
await channel.send(embed=embed)
print(f"Posted update for {codename} in Discord")
return
await self.channels['other'].send(embed=embed)
print(f"Posted update for {codename} in Discord")
def __init__(self):
utils.log("Started service")
revision = utils.get_revision()
utils.log("Board revision: {}".format(revision))
if revision is not None:
utils.set_property_setting('revision', revision)
max_ram = utils.get_max_ram()
utils.log("RAM size: {}MB".format(max_ram))
utils.set_property_setting('max_ram', max_ram)
board_type = utils.get_type()
utils.log("Board type: {}".format(board_type))
if board_type is not None:
utils.set_property_setting('type', board_type)
try:
utils.maybe_init_settings()
except IOError:
utils.log_exception()
self.monitor = MyMonitor(updated_settings_callback=self.apply_config)
while (not xbmc.abortRequested):
xbmc.sleep(1000)
def generate_message(update: dict):
"""
generates telegram message
"""
android = update['android']
codename = update['codename']
device = update['device']
download = update['download']
filename = update['filename']
filesize = update['size']
version = update['version']
branch = get_branch(version).capitalize()
region = get_region(filename, codename, version)
rom_type = get_type(filename)
codename = codename.split('_')[0]
message = f"New {branch} {rom_type} update available!\n"
message += f"*Device:* {device} \n" \
f"*Codename:* #{codename} \n" \
f"*Region:* {region} \n" \
f"*Version:* `{version}` \n" \
f"*Android:* {android} \n" \
f"*Size*: {filesize} \n" \
f"*Download*: [Here]({download})\n\n" \
f"[Latest Updates](https://xiaomifirmwareupdater.com/miui/{codename}) - " \
f"[All Updates](https://xiaomifirmwareupdater.com/archive/miui/{codename})\n" \
"@MIUIUpdatesTracker | @XiaomiFirmwareUpdater"
return message
def get_resources(pe):
resources = []
if hasattr(pe, 'DIRECTORY_ENTRY_RESOURCE'):
count = 1
for resource_type in pe.DIRECTORY_ENTRY_RESOURCE.entries:
try:
resource = {}
if resource_type.name is not None:
name = str(resource_type.name)
else:
name = str(pefile.RESOURCE_TYPE.get(resource_type.struct.Id, "UNKNOWN"))
if name is None:
name = str(resource_type.struct.Id)
if hasattr(resource_type, 'directory'):
for resource_id in resource_type.directory.entries:
if hasattr(resource_id, 'directory'):
for resource_lang in resource_id.directory.entries:
data = pe.get_data(resource_lang.data.struct.OffsetToData,
resource_lang.data.struct.Size)
entropy = get_entropy(data)
filetype = get_type(data)
md5 = get_md5(data)
sha256 = get_sha256(data)
language = pefile.LANG.get(resource_lang.data.lang, None)
language_desc = LCID.get(resource_lang.id, 'unknown language')
sublanguage = pefile.get_sublang_name_for_lang(
resource_lang.data.lang, resource_lang.data.sublang)
offset = ('%-8s' % hex(resource_lang.data.struct.OffsetToData)).strip()
size = ('%-8s' % hex(resource_lang.data.struct.Size)).strip()
resource = [
count, name, offset, md5, sha256, size, filetype, entropy, language,
sublanguage, language_desc
]
# Dump resources if requested
if self.dump and pe == self.pe:
if self.dump:
folder = self.dump
else:
folder = tempfile.mkdtemp()
resource_path = path.join(folder, '{0}_{1}_{2}'.format(
self.sha256, offset, name))
resource.append(resource_path)
with open(resource_path, 'wb') as resource_handle:
resource_handle.write(data)
resources.append(resource)
count += 1
except Exception as e:
log.error(e)
continue
return resources
def visit(self, node: StaticCallNode, scope: Scope):
typex = self.current_type
meth = self._get_method(typex, node.id, node.pos)
if not isinstance(meth, MethodError):
self._check_args(meth, scope, node.args, node.pos)
return get_type(meth.return_type, typex)
def visit(self, node: CallNode, scope: Scope):
stype = self.visit(node.obj, scope)
meth = self._get_method(stype, node.id, node.pos)
if not isinstance(meth, MethodError):
self._check_args(meth, scope, node.args, node.pos)
return get_type(meth.return_type, stype)
def benchmark_onnxruntime(path_to_model,
repeat=1000,
number=1,
warmup=100,
quantize=False):
"""
Parameters
----------
path_to_model: str or onnx.ModelProto
Path to an onnx model.
repeat: int
Repetition of experiment. Default: 1000
number: int
Number of forward passes in each experiment. Default: 1
warmup: int
Number of disregarded experiments. Default: 100
quantize: bool
Dynamically quantize the model with default parameters.
Returns
-------
info: dict
Information about the size and min, max, mean, std of the time
of the experiments.
"""
assert repeat >= 2 * warmup
if quantize:
import onnx
from onnx import version_converter
from onnxruntime.quantization import quantize_dynamic
orig_model = onnx.load(path_to_model)
if orig_model.opset_import[0].version < 11:
converted_model = version_converter.convert_version(orig_model, 11)
path_to_model = '/tmp/model_conv.onnx'
with open(path_to_model, 'wb') as f:
f.write(converted_model.SerializeToString())
del orig_model, converted_model
path_to_quant_model = "/tmp/model_quant.onnx"
model = quantize_dynamic(path_to_model, path_to_quant_model)
size = os.path.getsize(path_to_quant_model)
sess = ort.InferenceSession(path_to_quant_model)
else:
size = os.path.getsize(path_to_model)
sess = ort.InferenceSession(path_to_model)
inputs = {
x.name: np.random.randn(*get_shape(x)).astype(get_type(x))
for x in sess.get_inputs()
}
def _benchmark():
output = sess.run(None, inputs)
res = dict(size=size, input_size=[tuple(x.shape) for x in inputs.values()])
res.update(benchmark_speed(_benchmark, repeat, number, warmup))
return res
def visit(self, node: InstantiateNode, scope: Scope):
try:
type_ = self.context.get_type(node.lex, node.pos)
except SemanticError:
type_ = ErrorType()
error_text = TypesError.NEW_UNDEFINED_CLASS % node.lex
self.errors.append(TypesError(error_text, *node.pos))
return get_type(type_, self.current_type)
def visit(self, node: AssignNode, scope: Scope):
vinfo = self.find_variable(scope, node.id)
vtype = get_type(vinfo.type, self.current_type)
typex = self.visit(node.expr, scope)
if not typex.conforms_to(vtype):
error_text = TypesError.UNCONFORMS_TYPE % (typex.name, node.id,
vtype.name)
self.errors.append(TypesError(error_text, *node.pos))
return typex
def visit(self, node: VarDeclarationNode, scope: Scope):
vtype = self._get_type(node.type, node.type_pos)
vtype = get_type(vtype, self.current_type)
if node.expr != None:
typex = self.visit(node.expr, scope)
if not typex.conforms_to(vtype):
error_text = TypesError.UNCONFORMS_TYPE % (typex.name, node.id,
vtype.name)
self.errors.append(TypesError(error_text, *node.type_pos))
return typex
return vtype
def visit(self, node: AttrDeclarationNode, scope: Scope):
attr = self.current_type.get_attribute(node.id, node.pos)
vartype = get_type(attr.type, self.current_type)
self.current_index = attr.index
typex = self.visit(node.expr, scope)
self.current_index = None
if not typex.conforms_to(vartype):
error_text = TypesError.ATTR_TYPE_ERROR % (typex.name, attr.name,
vartype.name)
self.errors.append(TypesError(error_text, *node.pos))
return ErrorType()
return typex
def compare(header1: str, header2: str) -> float:
"""Computes a correlation between two columns in the data set, given their headers"""
list1: list = dataset_cols[header1]
list2: list = dataset_cols[header2]
assert len(list1) == len(list2)
type1: str = get_type(header=header1, col_types=col_types)
type2: str = get_type(header=header2, col_types=col_types)
stat = None
if type1 == NOMINAL_TYPE and type2 == NOMINAL_TYPE:
stat: float = run_contingency(list1=list1, list2=list2)
elif type1 == NOMINAL_TYPE and type2 == NUMERIC_TYPE:
stat: float = anova(numbers=list2, categories=list1)
elif type2 == NOMINAL_TYPE and type1 == NUMERIC_TYPE:
stat: float = anova(numbers=list1, categories=list2)
elif type1 == NUMERIC_TYPE and type2 == NUMERIC_TYPE:
stat: float = run_corr(list1=list1, list2=list2)
else:
print("ERROR: Non-specified type at " + header1 + " x " + header2)
exit(1)
return stat
def visit(self, node: BaseCallNode, scope: Scope):
obj = self.visit(node.obj, scope)
typex = self._get_type(node.type, node.type_pos)
if not obj.conforms_to(typex):
error_text = TypesError.INCOMPATIBLE_TYPES_DISPATCH % (typex.name,
obj.name)
self.errors.append(TypesError(error_text, *node.type_pos))
return ErrorType()
meth = self._get_method(typex, node.id, node.pos)
if not isinstance(meth, MethodError):
self._check_args(meth, scope, node.args, node.pos)
return get_type(meth.return_type, typex)
def club(user, chat, args, obj):
club = brawl.get_club(args[0])
if not club.get("name"):
sender.sendMessage("Информация о клубе с тегом {} не найдена!".format(args[0]))
return
msg = bot.get_messages("club", n="\n",
name=club.get("name"),
tag=args[0],
requiredTrophies=club.get("requiredTrophies"),
trophies=club.get("trophies"),
members=len(club.get("members")),
description=club.get("description"),
type=get_type(club.get("type"))
)
kb = VkKeyboard(inline=True)
kb.add_button(bot.get_messages("btn-club"), color=VkKeyboardColor.NEGATIVE, payload={
"action": "members",
"tag": args[0]
})
chat.sendKeyboard(msg, kb.get_keyboard())
def visit(self, node: FuncDeclarationNode, scope: Scope):
parent = self.current_type.parent
self.current_method = method = self.current_type.get_method(
node.id, node.pos)
if parent is not None:
try:
old_meth = parent.get_method(node.id, node.pos)
if old_meth.return_type.name != method.return_type.name:
error_text = SemanticError.WRONG_SIGNATURE_RETURN % (
node.id, method.return_type.name,
old_meth.return_type.name)
self.errors.append(
SemanticError(error_text, *node.type_pos))
if len(method.param_names) != len(old_meth.param_names):
error_text = SemanticError.WRONG_NUMBER_PARAM % node.id
self.errors.append(SemanticError(error_text, *node.pos))
for (name, param), type1, type2 in zip(node.params,
method.param_types,
old_meth.param_types):
if type1.name != type2.name:
error_text = SemanticError.WRONG_SIGNATURE_PARAMETER % (
name, type1.name, type2.name)
self.errors.append(
SemanticError(error_text, *param.pos))
except SemanticError:
pass
result = self.visit(node.body, scope)
return_type = get_type(method.return_type, self.current_type)
if not result.conforms_to(return_type):
error_text = TypesError.RETURN_TYPE_ERROR % (result.name,
return_type.name)
self.errors.append(TypesError(error_text, *node.type_pos))
#!/usr/bin/env python
import csv
import sys
from utils import get_type
reader = csv.reader(sys.stdin)
# Skip first row
next(reader, None)
for entry in reader:
Latitude = str(entry[21])
if Latitude == '' or Latitude is None:
label = 'NULL'
Latitude = 'NULL'
else:
# There is no outliers showing in boxplot so we make every non-missing values valid
label = 'VALID'
print('%s\t%s,latitude,%s' % (Latitude, get_type(Latitude), label))
import csv
import sys
from utils import get_type
import datetime
reader = csv.reader(sys.stdin)
# Skip first row
next(reader, None)
for entry in reader:
RPT_DT = str(entry[5])
if RPT_DT == '' or RPT_DT is None:
label = 'NULL'
RPT_DT = 'NULL'
else:
try:
# define upper bound and lower bound to exclude outliers
dt = datetime.datetime.strptime(RPT_DT, '%m/%d/%Y')
lb = datetime.datetime.strptime('01/01/2006', '%m/%d/%Y')
ub = datetime.datetime.strptime('12/31/2015', '%m/%d/%Y')
if dt > ub or dt < lb:
label = 'INVALID'
else:
label = 'VALID'
except ValueError:
label = 'INVALID'
print('%s\t%s,date,%s' % (RPT_DT, get_type(RPT_DT), label))
values.append(col_value)
data = values
#query = """INSERT INTO biblio_metadata"""
query = ""
for i, row in enumerate(data):
if i == 0:
#query += ' VALUES '
continue
elif i > 100:
break
idd = int(row[0])
title = str_sql_refactor(row[2])
khpl = str_sql_refactor(row[1])
author = str_sql_refactor(row[4])
typee = get_type(row[7])
metadata = """<?xml version="1.0" encoding="UTF-8"?>
<record
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd"
xmlns="http://www.loc.gov/MARC21/slim">
<leader>" "</leader>
<datafield tag="999" ind1=" " ind2=" ">
<subfield code="c">%d</subfield>
<subfield code="d">%d</subfield>
</datafield>
<datafield tag="082" ind1=" " ind2=" ">
<subfield code="a">%s</subfield>
<subfield code="b">" "</subfield>
</datafield>
#!/usr/bin/env python
import csv
import sys
from utils import get_type
reader = csv.reader(sys.stdin)
# Skip first row
next(reader, None)
for entry in reader:
PARKS_NM = str(entry[17])
if PARKS_NM == '' or PARKS_NM is None or PARKS_NM == 'N/A':
label = 'NULL'
PARKS_NM = 'NULL'
else:
label = 'VALID'
print('%s\t%s,name,%s' % (PARKS_NM, get_type(PARKS_NM), label))
#!/usr/bin/env python
import csv
import sys
from utils import get_type
reader = csv.reader(sys.stdin)
# Skip first row
next(reader, None)
# All possible pre-defined values
valid_list = ['ATTEMPTED', 'COMPLETED']
for entry in reader:
CRM_ATPT_CPTD_CD = str(entry[10])
if CRM_ATPT_CPTD_CD in valid_list:
label = 'VALID'
elif CRM_ATPT_CPTD_CD == '' or CRM_ATPT_CPTD_CD is None:
label = 'NULL'
CRM_ATPT_CPTD_CD = 'NULL'
else:
label = 'INVALID'
print('%s\t%s,flag,%s' %
(CRM_ATPT_CPTD_CD, get_type(CRM_ATPT_CPTD_CD), label))
#!/usr/bin/env python
import csv
import sys
from utils import get_type
reader = csv.reader(sys.stdin)
# Skip first row
next(reader, None)
valid_list = ['BRONX', 'BROOKLYN', 'MANHATTAN', 'QUEENS', 'STATEN ISLAND']
for entry in reader:
BORO_NM = str(entry[13])
if BORO_NM in valid_list:
label = 'VALID'
elif BORO_NM == '' or BORO_NM is None:
label = 'NULL'
BORO_NM = 'NULL'
else:
label = 'INVALID'
print('%s\t%s,borough,%s' % (BORO_NM, get_type(BORO_NM), label))
#!/usr/bin/env python
import csv
import sys
from utils import get_type
reader = csv.reader(sys.stdin)
# Skip first row
next(reader, None)
for entry in reader:
HADEVELOPT = str(entry[18])
if HADEVELOPT == '' or HADEVELOPT is None or HADEVELOPT == 'N/A':
label = 'NULL'
HADEVELOPT = 'NULL'
else:
label = 'VALID'
print('%s\t%s,address,%s' % (HADEVELOPT, get_type(HADEVELOPT), label))
#!/usr/bin/env python
import csv
import sys
from utils import get_type
reader = csv.reader(sys.stdin)
# Skip first row
next(reader, None)
# All possible pre-defined values
valid_list = ['INSIDE', 'OPPOSITE OF', 'OUTSIDE', 'REAR OF', 'FRONT OF']
for entry in reader:
LOC_OF_OCCUR_DESC = str(entry[15])
if LOC_OF_OCCUR_DESC in valid_list:
label = 'VALID'
elif LOC_OF_OCCUR_DESC == '' or LOC_OF_OCCUR_DESC == ' ' or LOC_OF_OCCUR_DESC is None:
label = 'NULL'
LOC_OF_OCCUR_DESC = 'NULL'
else:
label = 'INVALID'
print('%s\t%s,description,%s' %
(LOC_OF_OCCUR_DESC, get_type(LOC_OF_OCCUR_DESC), label))
#!/usr/bin/env python
import csv
import sys
from utils import get_type
reader = csv.reader(sys.stdin)
# Skip first row
next(reader, None)
for entry in reader:
PD_DESC = str(entry[9])
if PD_DESC == '' or PD_DESC is None or PD_DESC == 'N/A':
label = 'NULL'
PD_DESC = 'NULL'
else:
label = 'VALID'
print('%s\t%s,description,%s' % (PD_DESC, get_type(PD_DESC), label))
#!/usr/bin/env python
import csv
import sys
from utils import get_type
reader = csv.reader(sys.stdin)
# Skip first row
next(reader, None)
for entry in reader:
Lat_Lon = str(entry[23])
if Lat_Lon == '' or Lat_Lon is None:
label = 'NULL'
Lat_Lon = 'NULL'
else:
# Check whether value pairs are corresponding correctly to original values
Latitude = str(entry[21])
Longitude = str(entry[22])
if Lat_Lon.split(',')[0][1:] == Latitude and Lat_Lon.split(
',')[1][1:-1] == Longitude:
label = 'VALID'
else:
label = 'INVALID'
print('%s\t%s,coordinate,%s' % (Lat_Lon, get_type(Lat_Lon), label))
import csv
import sys
from utils import get_type
reader = csv.reader(sys.stdin)
# Skip first row
next(reader, None)
# All possible pre-defined values
valid_list = [
'AMTRACK', 'CONRAIL', 'DEPT OF CORRECTIONS', 'DISTRICT ATTORNEY OFFICE',
'FIRE DEPT (FIRE MARSHAL)', 'HEALTH & HOSP CORP', 'LONG ISLAND RAILRD',
'METRO NORTH', 'N.Y. HOUSING POLICE', 'N.Y. POLICE DEPT',
'N.Y. STATE PARKS', 'N.Y. STATE POLICE', 'N.Y. TRANSIT POLICE',
'NEW YORK CITY SHERIFF OFFICE', 'NYC DEPT ENVIRONMENTAL PROTECTION',
'NYC PARKS', 'NYS DEPT ENVIRONMENTAL CONSERVATION',
'NYS DEPT TAX AND FINANCE', 'OTHER', 'POLICE DEPT NYC', 'PORT AUTHORITY',
'SEA GATE POLICE DEPT', 'STATN IS RAPID TRANS', 'TRI-BORO BRDG TUNNL',
'U.S. PARK POLICE'
]
for entry in reader:
JURIS_DESC = str(entry[12])
if JURIS_DESC in valid_list:
label = 'VALID'
elif JURIS_DESC == '' or JURIS_DESC is None:
label = 'NULL'
JURIS_DESC = 'NULL'
else:
label = 'INVALID'
print('%s\t%s,jurisdiction,%s' % (JURIS_DESC, get_type(JURIS_DESC), label))
#!/usr/bin/env python
import csv
import sys
from utils import get_type
import datetime
reader = csv.reader(sys.stdin)
# Skip first row
next(reader, None)
for entry in reader:
CMPLNT_TO_TM = str(entry[4])
if CMPLNT_TO_TM == '' or CMPLNT_TO_TM is None:
label = 'NULL'
CMPLNT_TO_TM = 'NULL'
else:
try:
# If the transformation fails, then it is invalid
datetime.datetime.strptime(CMPLNT_TO_TM, '%H:%M:%S')
label = 'VALID'
except ValueError:
label = 'INVALID'
print('%s\t%s,time,%s' % (CMPLNT_TO_TM, get_type(CMPLNT_TO_TM), label))
def add_malware():
'''
Adds a sample to the repository. Performs hashing and filemagic
analysis of the uploaded sample.
@tags : comma seperated tags list
@file : binary sample stream
returns : JSON status message
'''
try:
with timeout(Config().api.timeout * 60, exception=RuntimeError):
tags = request.forms.get('tags').split(',')
data = request.files.file
data.file.seek(0)
filename = data.filename
sampleData = data.file.read()
logging.debug('[%s] Generating hashes' % sampleEntry)
md5 = hashlib.md5(sampleData).hexdigest()
sha1 = hashlib.sha1(sampleData).hexdigest()
sha256 = hashlib.sha256(sampleData).hexdigest()
sha512 = hashlib.sha512(sampleData).hexdigest()
filetype = get_type(sampleData)
key = {'md5': md5}
logging.debug('Quering database for already existing file (hash=%s)'
% md5)
existing = db.fs.files.find_one({'md5': md5})
upload_sample = True
if existing:
logging.info('Sample already exists')
logging.info('Verifying contents')
if not md5 == existing['md5']:
logging.warning('Checksum not matching')
upload_sample = True
else:
logging.info('Checksum matching')
upload_sample = False
else:
upload_sample = True
if upload_sample:
logging.debug('Uploading sample')
new = fs.new_file(filename=filename, sha1=sha1,
sha256=sha256, sha512=sha512,
filetype=filetype)
for chunk in get_chunks(sampleData):
logging.debug('writing chunk')
new.write(chunk)
new.close()
logging.info('Uploaded sample')
add_tags(key=key, tags=tags)
logging.debug('Reclaiming memory')
del sampleData
response.content_type = 'application/json'
return jsonize({'message': 'added'})
except RuntimeError:
response.content_type = 'application/json'
return (jsonize({'error': 'timeout'}), 504)
#!/usr/bin/env python
import csv
import sys
from utils import get_type
reader = csv.reader(sys.stdin)
# Skip first row
next(reader, None)
for entry in reader:
PREM_TYP_DESC = str(entry[16])
if PREM_TYP_DESC == '' or PREM_TYP_DESC is None or PREM_TYP_DESC == 'N/A':
label = 'NULL'
PREM_TYP_DESC = 'NULL'
else:
label = 'VALID'
print('%s\t%s,description,%s' %
(PREM_TYP_DESC, get_type(PREM_TYP_DESC), label))
numOfFiles = len(archive.namelist())
for (index, sampleEntry) in enumerate(archive.namelist()):
logging.info('[%s] [%s] Got entry (%s/%s)' %
(zipname, sampleEntry, index + 1, numOfFiles))
sample = archive.open(sampleEntry, 'r')
sampleData = sample.read()
logging.debug('[%s] [%s] Generating hashes' % (zipname, sampleEntry))
md5 = hashlib.md5(sampleData).hexdigest()
sha1 = hashlib.sha1(sampleData).hexdigest()
sha256 = hashlib.sha256(sampleData).hexdigest()
sha512 = hashlib.sha512(sampleData).hexdigest()
filetype = get_type(sampleData)
logging.debug(
'[%s] [%s] Quering database for already existing file (hash=%s)' %
(zipname, sampleEntry, sha256))
existing = db.fs.files.find_one({'sha256': sha256})
upload_sample = True
if existing:
logging.info('[%s] [%s] Sample already exists' %
(zipname, sampleEntry))
logging.info('[%s] [%s] Verifying contents' %
(zipname, sampleEntry))
if not md5 == existing['md5']:
logging.warning('[%s] [%s] Checksum not matching' %
(zipname, sampleEntry))
enumerate(db.fs.files.find({'filetype': {'$exists': False}},
timeout=False)):
try:
logger.info('[%s] Processing sample %s' % (sampleno,
sample['sha256']))
samplekey = {'sha256': sample['sha256']}
# download sample file
logger.debug('[%s] Downloading data' % sampleno)
data = get_file(db, sha256=sample['sha256'])
# Do analysis
logger.debug('[%s] Analysing' % sampleno)
file_type = get_type(data)
# Store results
logger.debug('[%s] Storing results into MongoDB'
% sampleno)
if file_type:
db.fs.files.update(samplekey,
{'$set': {'filetype': file_type}},
upsert=True)
# delete sample file
logger.debug('[%s] Deleting temporary data' % sampleno)
del samplekey
del data
numOfFiles = len(archive.namelist())
for (index, sampleEntry) in enumerate(archive.namelist()):
logging.info('[%s] [%s] Got entry (%s/%s)' % (zipname, sampleEntry,
index + 1, numOfFiles))
sample = archive.open(sampleEntry, 'r')
sampleData = sample.read()
logging.debug('[%s] [%s] Generating hashes' % (zipname,
sampleEntry))
md5 = hashlib.md5(sampleData).hexdigest()
sha1 = hashlib.sha1(sampleData).hexdigest()
sha256 = hashlib.sha256(sampleData).hexdigest()
sha512 = hashlib.sha512(sampleData).hexdigest()
filetype = get_type(sampleData)
logging.debug('[%s] [%s] Quering database for already existing file (hash=%s)'
% (zipname, sampleEntry, sha256))
existing = db.fs.files.find_one({'sha256': sha256})
upload_sample = True
if existing:
logging.info('[%s] [%s] Sample already exists' % (zipname,
sampleEntry))
logging.info('[%s] [%s] Verifying contents' % (zipname,
sampleEntry))
if not md5 == existing['md5']:
logging.warning('[%s] [%s] Checksum not matching'
% (zipname, sampleEntry))
upload_sample = True
#!/usr/bin/env python
import csv
import sys
from utils import get_type
reader = csv.reader(sys.stdin)
# Skip first row
next(reader, None)
for entry in reader:
PD_CD = str(entry[8])
# Check whether this is a three-digit code
if len(PD_CD) == 3:
label = 'VALID'
elif PD_CD == '' or PD_CD is None:
label = 'NULL'
PD_CD = 'NULL'
else:
label = 'INVALID'
print('%s\t%s,code,%s' % (PD_CD, get_type(PD_CD), label))
