def test_get_args_and_env_run_script():
msg = {'key': '1/2/3', 'job_cmd': ['custom.py', 'arg1'],
'auth': 'authstring'}
result = get_args_and_env(msg)
expected_auth = codecs.encode(to_bytes('1/2/3:authstring'), 'hex')
assert len(result) == 2
assert result[0] == ['custom.py', 'arg1']
assert result[1] == {
'SHUB_JOBAUTH': to_native_str(expected_auth),
'SHUB_JOBKEY': '1/2/3',
'SHUB_JOBNAME': 'custom.py',
'SHUB_JOB_TAGS': ''}
add_fields = {'tags': ['tagA', 'tagB'], 'api_url': 'some-api-url'}
msg.update(add_fields)
result1 = get_args_and_env(msg)
assert len(result1) == 2
assert result1[1]['SHUB_APIURL'] == 'some-api-url'
assert result1[1]['SHUB_JOB_TAGS'] == 'tagA,tagB'
def test_get_args_and_env_run_script():
msg = {
'key': '1/2/3',
'job_cmd': ['custom.py', 'arg1'],
'auth': 'authstring'
}
result = get_args_and_env(msg)
expected_auth = codecs.encode(to_bytes('1/2/3:authstring'), 'hex_codec')
assert len(result) == 2
assert result[0] == ['custom.py', 'arg1']
assert result[1] == {
'SHUB_JOBAUTH': to_native_str(expected_auth),
'SHUB_JOBKEY': '1/2/3',
'SHUB_JOBNAME': 'custom.py',
'SHUB_JOB_TAGS': ''
}
add_fields = {'tags': ['tagA', 'tagB'], 'api_url': 'some-api-url'}
msg.update(add_fields)
result1 = get_args_and_env(msg)
assert len(result1) == 2
assert result1[1]['SHUB_APIURL'] == 'some-api-url'
assert result1[1]['SHUB_JOB_TAGS'] == 'tagA,tagB'
def test_get_args_and_env_run_spider():
msg = {
'key': '1/2/3',
'spider': 'test',
'spider_type': 'auto',
'auth': 'auths',
'spider_args': {
'arg1': 'val1',
'arg2': 'val2'
},
'settings': {
'SETTING1': 'VAL1',
'SETTING2': 'VAL2'
}
}
result = get_args_and_env(msg)
expected_auth = codecs.encode(to_bytes('1/2/3:auths'), 'hex_codec')
assert len(result) == 2
assert result[0] == [
'scrapy', 'crawl', 'test', '-a', 'arg1=val1', '-a', 'arg2=val2', '-s',
'SETTING1=VAL1', '-s', 'SETTING2=VAL2'
]
assert result[1] == {
'SCRAPY_JOB': '1/2/3',
'SCRAPY_PROJECT_ID': '1',
'SCRAPY_SPIDER': 'test',
'SHUB_JOBAUTH': to_native_str(expected_auth),
'SHUB_JOBKEY': '1/2/3',
'SHUB_JOBNAME': 'test',
'SHUB_JOB_TAGS': '',
'SHUB_SPIDER_TYPE': 'auto'
}
add_fields = {'tags': ['tagA', 'tagB'], 'api_url': 'some-api-url'}
msg.update(add_fields)
result1 = get_args_and_env(msg)
assert len(result1) == 2
assert result1[1]['SHUB_APIURL'] == 'some-api-url'
assert result1[1]['SHUB_JOB_TAGS'] == 'tagA,tagB'
def test_get_args_and_env_run_spider():
msg = {'key': '1/2/3', 'spider': 'test', 'spider_type': 'auto',
'auth': 'auths', 'spider_args': {'arg1': 'val1', 'arg2': 'val2'},
'settings': {'SETTING1': 'VAL1', 'SETTING2': 'VAL2'}}
result = get_args_and_env(msg)
expected_auth = codecs.encode(to_bytes('1/2/3:auths'), 'hex')
assert len(result) == 2
assert result[0] == ['scrapy', 'crawl', 'test', '-a', 'arg1=val1',
'-a', 'arg2=val2', '-s', 'SETTING1=VAL1', '-s',
'SETTING2=VAL2']
assert result[1] == {'SCRAPY_JOB': '1/2/3',
'SCRAPY_PROJECT_ID': '1',
'SCRAPY_SPIDER': 'test',
'SHUB_JOBAUTH': to_native_str(expected_auth),
'SHUB_JOBKEY': '1/2/3',
'SHUB_JOBNAME': 'test',
'SHUB_JOB_TAGS': '',
'SHUB_SPIDER_TYPE': 'auto'}
add_fields = {'tags': ['tagA', 'tagB'], 'api_url': 'some-api-url'}
msg.update(add_fields)
result1 = get_args_and_env(msg)
assert len(result1) == 2
assert result1[1]['SHUB_APIURL'] == 'some-api-url'
assert result1[1]['SHUB_JOB_TAGS'] == 'tagA,tagB'
def _jobauth(msg):
auth_data = to_bytes('{0[key]}:{0[auth]}'.format(msg))
return to_native_str(codecs.encode(auth_data, 'hex_codec'))
# -*- coding: utf-8 -*-
import codecs
import os
import shutil
import tempfile
import pytest
TEMP_DIR = tempfile.mkdtemp()
SHUB_FIFO_PATH = os.path.join(TEMP_DIR, 'scrapinghub')
os.environ['SHUB_FIFO_PATH'] = SHUB_FIFO_PATH
from sh_scrapy.compat import to_native_str, to_bytes
TEST_AUTH = to_native_str(codecs.encode(to_bytes('1/2/3:authstr'),
'hex_codec'))
@pytest.fixture(scope='session', autouse=True)
def clean_shub_fifo_path():
global TEMP_DIR
try:
yield
finally:
shutil.rmtree(TEMP_DIR)
@pytest.fixture(autouse=True)
def set_jobkeyenvironment(monkeypatch):
monkeypatch.setenv('SHUB_JOBKEY', '1/2/3')
monkeypatch.setenv('SCRAPY_JOB', '1/2/3')
def test_to_bytes_errors_argument():
assert to_bytes(u'a\ufffdb', 'latin-1', errors='replace') == b'a?b'
def test_to_bytes_a_strange_object_should_raise_TypeError():
with pytest.raises(TypeError):
to_bytes(pytest)
def test_to_bytes_a_regular_bytes_to_bytes_should_return_the_same_object():
assert to_bytes(b'lel\xf1e') == b'lel\xf1e'
def test_to_bytes_a_unicode_object_to_a_latin_1_encoded_string():
assert to_bytes(u'\xa3 49', 'latin-1') == b'\xa3 49'
def test_to_bytes_a_unicode_object_to_an_utf_8_encoded_string():
assert to_bytes(u'\xa3 49') == b'\xc2\xa3 49'
def test_to_bytes_a_unicode_object_to_an_utf_8_encoded_string():
assert to_bytes(u'\xa3 49') == b'\xc2\xa3 49'
def test_jobauth():
msg = {'key': '1/2/3', 'auth': 'authstring'}
expected = codecs.encode(to_bytes('1/2/3:authstring'), 'hex_codec')
assert _jobauth(msg) == to_native_str(expected)
def test_jobauth():
msg = {'key': '1/2/3', 'auth': 'authstring'}
expected = codecs.encode(to_bytes('1/2/3:authstring'), 'hex')
assert _jobauth(msg) == to_native_str(expected)
def test_to_bytes_a_unicode_object_to_a_latin_1_encoded_string():
assert to_bytes(u'\xa3 49', 'latin-1') == b'\xa3 49'
def test_to_bytes_errors_argument():
assert to_bytes(u'a\ufffdb', 'latin-1', errors='replace') == b'a?b'
def test_to_bytes_a_strange_object_should_raise_TypeError():
with pytest.raises(TypeError):
to_bytes(pytest)
def test_to_bytes_a_regular_bytes_to_bytes_should_return_the_same_object():
assert to_bytes(b'lel\xf1e') == b'lel\xf1e'
import os
import sys
import mock
import pytest
import codecs
from sh_scrapy.hsref import _HubstorageRef
from sh_scrapy.compat import to_native_str, to_bytes
TEST_AUTH = to_native_str(codecs.encode(to_bytes('1/2/3:authstr'), 'hex'))
def test_init_disabled():
hsref = _HubstorageRef()
assert not hsref._client
assert not hsref._project
assert not hsref._job
assert not hsref.enabled
assert not hasattr(hsref, 'jobkey')
assert not hsref._projectid
assert not hsref._spiderid
assert not hsref._jobcounter
@pytest.fixture
@mock.patch.dict(os.environ, {'SHUB_JOBKEY': '1/2/3'})
def hsref():
return _HubstorageRef()
@pytest.fixture
def hsc_class(monkeypatch):
def _jobauth(msg):
auth_data = to_bytes('{0[key]}:{0[auth]}'.format(msg))
return to_native_str(codecs.encode(auth_data, 'hex_codec'))